code cleanup, fixhand examples autoload, change to youtube

.gitignore

@@ -1,4 +1,4 @@
-settings.json
+.vscode/
 sbatch/err/
 sbatch/out/
 __pycache__/

app.py

@@ -473,13 +473,10 @@ def visualize_ref(ref):
         img = ref["background"][..., :3]
 
     # viualization
-    # img = cv2.resize(img, inpainted.shape[::-1], interpolation=cv2.INTER_AREA)
     mask = inpainted < 128
-    # img = img.astype(np.int32)
-    # img[mask, :] = img[mask, :] - 50
-    # img[np.any(img<0, axis=-1)]=0
-    # img = img.astype(np.uint8)
     img = mask_image(img, mask)
+    if inpaint_mask.sum() == 0:
+        gr.Warning("Run botton not enabled? Please try again.", duration=10)
     return img, inpaint_mask
 
 def get_kps(img, keypoints, side: Literal["right", "left"], evt: gr.SelectData):
@@ -581,11 +578,6 @@ def process_crop(img, crop_coord, evt:gr.SelectData):
             cropped_vis[:,:,-1] = alpha
     else:
         gr.Error("Something is wrong", duration=3)
-    # cropped_editor = {
-    #     "background": cropped,
-    #     "composite": cropped,
-    #     "layers": [cropped_vis]
-    # }
     return crop_coord, cropped, cropped_vis
 
 def disable_crop(crop_coord):
@@ -653,8 +645,6 @@ def ready_sample(img_cropped, inpaint_mask, keypts, keypts_np):
             keypts[0] = np.zeros((21, 2))
         elif len(keypts[0]) == 21:
             keypts[0] = np.array(keypts[0], dtype=np.float32)
-            # keypts[0][:, 0] = keypts[0][:, 0] + crop_coord[0][0]
-            # keypts[0][:, 1] = keypts[0][:, 1] + crop_coord[0][1]
         else:
             gr.Info("Number of right hand keypoints should be either 0 or 21.")
             return None, None
@@ -662,8 +652,6 @@ def ready_sample(img_cropped, inpaint_mask, keypts, keypts_np):
             keypts[1] = np.zeros((21, 2))
         elif len(keypts[1]) == 21:
             keypts[1] = np.array(keypts[1], dtype=np.float32)
-            # keypts[1][:, 0] = keypts[1][:, 0] + crop_coord[0][0]
-            # keypts[1][:, 1] = keypts[1][:, 1] + crop_coord[0][1]
         else:
             gr.Info("Number of left hand keypoints should be either 0 or 21.")
             return None, None
@@ -868,13 +856,7 @@ def flip_hand(
         return
     img["composite"] = img["composite"][:, ::-1, :]
     img["background"] = img["background"][:, ::-1, :]
-    img["layers"] = [layer[:, ::-1, :] for layer in img["layers"]]
-    # for comp in [pose_img, pose_manual_img, manual_kp_right, manual_kp_left, cond, auto_cond, manual_cond]:
-    #     if comp is not None:
-    #         if isinstance(comp, torch.Tensor):
-    #             comp = comp.flip(-1)
-    #         else:
-    #             comp = comp[:, ::-1, :]    
+    img["layers"] = [layer[:, ::-1, :] for layer in img["layers"]] 
     if img_raw is not None:
         img_raw = img_raw[:, ::-1, :]
     pose_img = pose_img[:, ::-1, :]
@@ -889,12 +871,6 @@ def flip_hand(
         auto_cond = auto_cond.flip(-1)
     if manual_cond is not None:
         manual_cond = manual_cond.flip(-1)
-    # for comp in [keypts, auto_keypts, manual_keypts]:
-    #     if comp is not None:
-    #         if comp[:21, :].sum() != 0:
-    #             comp[:21, 0] = opts.image_size[1] - comp[:21, 0]
-    #         if comp[21:, :].sum() != 0:
-    #             comp[21:, 0] = opts.image_size[1] - comp[21:, 0]
     if keypts is not None:
         if keypts[:21, :].sum() != 0:
             keypts[:21, 0] = opts.image_size[1] - keypts[:21, 0]
@@ -980,6 +956,10 @@ def fix_clear_all():
 def enable_component(image1, image2):
     if image1 is None or image2 is None:
         return gr.update(interactive=False)
+    if isinstance(image1, np.ndarray) and image1.sum() == 0:
+        return gr.update(interactive=False)
+    if isinstance(image2, np.ndarray) and image2.sum() == 0:
+        return gr.update(interactive=False)
     if isinstance(image1, dict) and "background" in image1 and "layers" in image1 and "composite" in image1:
         if image1["background"] is None or (
             image1["background"].sum() == 0
@@ -1109,61 +1089,14 @@ example_ref_imgs = [
     [
         "sample_images/sample4.jpg",
     ],
-    # [
-    #     "sample_images/sample5.jpg",
-    # ],
     [
         "sample_images/sample6.jpg",
     ],
-    # [
-    #     "sample_images/sample7.jpg",
-    # ],
-    # [
-    #     "sample_images/sample8.jpg",
-    # ],
-    # [
-    #     "sample_images/sample9.jpg",
-    # ],
-    # [
-    #     "sample_images/sample10.jpg",
-    # ],
-    # [
-    #     "sample_images/sample11.jpg",
-    # ],
-    # ["pose_images/pose1.jpg"],
-    # ["pose_images/pose2.jpg"],
-    # ["pose_images/pose3.jpg"],
-    # ["pose_images/pose4.jpg"],
-    # ["pose_images/pose5.jpg"],
-    # ["pose_images/pose6.jpg"],
-    # ["pose_images/pose7.jpg"],
-    # ["pose_images/pose8.jpg"],
 ]
 example_target_imgs = [
-    # [
-    #     "sample_images/sample1.jpg",
-    # ],
-    # [
-    #     "sample_images/sample2.jpg",
-    # ],
-    # [
-    #     "sample_images/sample3.jpg",
-    # ],
-    # [
-    #     "sample_images/sample4.jpg",
-    # ],
     [
         "sample_images/sample5.jpg",
     ],
-    # [
-        # "sample_images/sample6.jpg",
-    # ],
-    # [
-    #     "sample_images/sample7.jpg",
-    # ],
-    # [
-    #     "sample_images/sample8.jpg",
-    # ],
     [
         "sample_images/sample9.jpg",
     ],
@@ -1174,40 +1107,22 @@ example_target_imgs = [
         "sample_images/sample11.jpg",
     ],
     ["pose_images/pose1.jpg"],
-    # ["pose_images/pose2.jpg"],
-    # ["pose_images/pose3.jpg"],
-    # ["pose_images/pose4.jpg"],
-    # ["pose_images/pose5.jpg"],
-    # ["pose_images/pose6.jpg"],
-    # ["pose_images/pose7.jpg"],
-    # ["pose_images/pose8.jpg"],
 ]
 fix_example_imgs = [
-    ["bad_hands/1.jpg"],  # "bad_hands/1_mask.jpg"],
-    # ["bad_hands/2.jpg"],  # "bad_hands/2_mask.jpg"],
-    ["bad_hands/3.jpg"],  # "bad_hands/3_mask.jpg"],
-    ["bad_hands/4.jpg"],  # "bad_hands/4_mask.jpg"],
-    ["bad_hands/5.jpg"],  # "bad_hands/5_mask.jpg"],
-    ["bad_hands/6.jpg"],  # "bad_hands/6_mask.jpg"],
-    ["bad_hands/7.jpg"],  # "bad_hands/7_mask.jpg"],
-    # ["bad_hands/8.jpg"],  # "bad_hands/8_mask.jpg"],
-    # ["bad_hands/9.jpg"],  # "bad_hands/9_mask.jpg"],
-    # ["bad_hands/10.jpg"],  # "bad_hands/10_mask.jpg"],
-    # ["bad_hands/11.jpg"],  # "bad_hands/11_mask.jpg"],
-    # ["bad_hands/12.jpg"],  # "bad_hands/12_mask.jpg"],
-    # ["bad_hands/13.jpg"],  # "bad_hands/13_mask.jpg"],
-    # ["bad_hands/14.jpg"],
-    # ["bad_hands/15.jpg"],
+    ["bad_hands/1.jpg"],
+    ["bad_hands/3.jpg"],
+    ["bad_hands/4.jpg"],
+    ["bad_hands/5.jpg"],
+    ["bad_hands/6.jpg"],
+    ["bad_hands/7.jpg"],
 ]
 fix_example_brush = [
-    ["bad_hands/1_composite.png"],# "bad_hands/1.jpg"],
-    ["bad_hands/3_composite.png"],# "bad_hands/3.jpg"],
-    ["bad_hands/4_composite.png"],# "bad_hands/4.jpg"],
-    ["bad_hands/5_composite.png"],# "bad_hands/5.jpg"],
-    ["bad_hands/6_composite.png"],# "bad_hands/6.jpg"],
-    ["bad_hands/7_composite.png"],# "bad_hands/7.jpg"],
-    # ["bad_hands/14_mask.jpg"],
-    # ["bad_hands/15_mask.jpg"],
+    ["bad_hands/1_composite.png"],
+    ["bad_hands/3_composite.png"],
+    ["bad_hands/4_composite.png"],
+    ["bad_hands/5_composite.png"],
+    ["bad_hands/6_composite.png"],
+    ["bad_hands/7_composite.png"],
 ]
 fix_example_kpts = [
     ["bad_hands/1_kpts.png", 3.0, 1224],
@@ -1217,9 +1132,20 @@ fix_example_kpts = [
     ["bad_hands/6_kpts.png", 3.0, 1348],
     ["bad_hands/7_kpts.png", 3.0, 42],
 ]
+fix_example_all = [
+    ["bad_hands/1.jpg", "bad_hands/1_composite.png", "bad_hands/1_kpts.png", 3.0, 1224],
+    ["bad_hands/3.jpg", "bad_hands/3_composite.png", "bad_hands/3_kpts.png", 1.0, 42],
+    ["bad_hands/4.jpg", "bad_hands/4_composite.png", "bad_hands/4_kpts.png", 2.0, 42],
+    ["bad_hands/5.jpg", "bad_hands/5_composite.png", "bad_hands/5_kpts.png", 3.0, 42],
+    ["bad_hands/6.jpg", "bad_hands/6_composite.png", "bad_hands/6_kpts.png", 3.0, 1348],
+    ["bad_hands/7.jpg", "bad_hands/7_composite.png", "bad_hands/7_kpts.png", 3.0, 42],
+]
 for i in range(len(fix_example_kpts)):
     npy_path = fix_example_kpts[i][0].replace("_kpts.png", ".npy")
     fix_example_kpts[i].append(npy_path)
+for i in range(len(fix_example_all)):
+    npy_path = fix_example_all[i][2].replace("_kpts.png", ".npy")
+    fix_example_all[i].append(npy_path)
 
 custom_css = """
 .gradio-container .examples img {
@@ -1248,6 +1174,18 @@ custom_css = """
 #kpts_examples table tr td:nth-child(4) {
     display: none !important;
 }
+#fix_examples_all table tr th:nth-child(4),
+#fix_examples_all table tr td:nth-child(4) {
+    display: none !important;
+}
+#fix_examples_all table tr th:nth-child(5),
+#fix_examples_all table tr td:nth-child(5) {
+    display: none !important;
+}
+#fix_examples_all table tr th:nth-child(6),
+#fix_examples_all table tr td:nth-child(6) {
+    display: none !important;
+}
 #repose_tutorial video {
     width: 70% !important;
     display: block;
@@ -1256,10 +1194,35 @@ custom_css = """
 }
 """
 
+tut1_custom = f"""
+<iframe style="width:100%; aspect-ratio: 12/9;"
+    src="https://www.youtube.com/embed/fQk7cOjSCVc"
+    title="Using your own image" frameborder="0"
+    allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+    allowfullscreen>
+</iframe>
+"""
+tut1_example = f"""
+<iframe style="width:100%; aspect-ratio: 12/9;"
+    src="https://www.youtube.com/embed/-Dq0XTYwTHA"
+    title="Using your own image" frameborder="0"
+    allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+    allowfullscreen>
+</iframe>
+"""
+tut2_example = f"""
+<iframe style="width:50%; aspect-ratio: 12/9; display:block; margin-left:auto; margin-right:auto;"
+    src="https://www.youtube.com/embed/y2CbzUG2uM0"
+    title="Using your own image" frameborder="0"
+    allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
+    allowfullscreen>
+</iframe>
+"""
+
 _HEADER_ = '''
 <div style="text-align: center;">
     <h1><b>FoundHand: Large-Scale Domain-Specific Learning for Controllable Hand Image Generation</b></h1>
-    <h2 style="color: #777777;">CVPR 2025 <span style="color: #990000; font-style: italic;">highlight</span></h2>
+    <h2 style="color: #777777;">CVPR 2025 <span style="color: #990000; font-style: italic;">Highlight</span></h2>
     <style>
         .link-spacing {
             margin-right: 20px;
@@ -1280,8 +1243,7 @@ _HEADER_ = '''
     <h3>
         <a href='https://arxiv.org/abs/2412.02690' target='_blank' class="link-spacing">Paper</a>
         <a href='https://ivl.cs.brown.edu/research/foundhand.html' target='_blank' class="link-spacing">Project Page</a>
-        <a href='' target='_blank' class="link-spacing">Code</a>
-        <a href='' target='_blank'>Model Weights</a>
+        <a href='' target='_blank' class="link-spacing">Code (Coming in June)</a>
     </h3>
     <p>Below are two important abilities of our model. First, we can automatically <b>fix malformed hand images</b>, following the user-provided target hand pose and area to fix. Second, we can <b>repose hand</b> given two hand images - one is the image to edit, and the other one provides target hand pose.</p>
 </div>
@@ -1323,21 +1285,23 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
             gr.Markdown("""<p style="text-align: center; font-size: 20px; font-weight: bold;">Tutorial Videos of Demo 1</p>""")
             with gr.Row(variant="panel"):
                 with gr.Column():
-                    gr.Video(
-                        "how_to_videos/subtitled_fix_hands_custom.mp4",
-                        label="Using your own image",
-                        autoplay=True,
-                        loop=True,
-                        show_label=True,
-                    )
+                    # gr.Video(
+                        # "how_to_videos/subtitled_fix_hands_custom.mp4",
+                        # label="Using your own image",
+                        # autoplay=True,
+                        # loop=True,
+                        # show_label=True,
+                    # )
+                    gr.HTML(tut1_custom)
                 with gr.Column():
-                    gr.Video(
-                        "how_to_videos/subtitled_fix_hands_example.mp4",
-                        label="Using our example image",
-                        autoplay=True,
-                        loop=True,
-                        show_label=True,
-                    )
+                    # gr.Video(
+                    #     "how_to_videos/subtitled_fix_hands_example.mp4",
+                    #     label="Using our example image",
+                    #     autoplay=True,
+                    #     loop=True,
+                    #     show_label=True,
+                    # )
+                    gr.HTML(tut1_example)
         
         # more options
         with gr.Accordion(label="More options", open=False):
@@ -1392,20 +1356,6 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 gr.Markdown(
                     """<p style="text-align: center;">Optionally crop the image.<br>(Click <b>top left</b> and <b>bottom right</b> of your desired bounding box around the hand)</p>"""
                 )
-                # fix_crop = gr.ImageEditor(
-                #     type="numpy",
-                #     sources=["upload", "webcam", "clipboard"],
-                #     label="Image crop",
-                #     show_label=True,
-                #     height=LENGTH,
-                #     width=LENGTH,
-                #     layers=False,
-                #     # crop_size="1:1",
-                #     transforms=(),
-                #     brush=False,
-                #     image_mode="RGBA",
-                #     container=False,
-                # )
                 fix_crop = gr.Image(
                     type="numpy",
                     sources=["upload", "webcam", "clipboard"],
@@ -1420,23 +1370,11 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 gr.Markdown(
                     """<p style="text-align: center;">💡 If you crop, the model can focus on more details of the cropped area. Square crops might work better than rectangle crops.</p>"""
                 )
-                # fix_tmp = gr.Image(
-                #     type="numpy",
-                #     label="tmp",
-                #     show_label=True,
-                #     height=LENGTH,
-                #     width=LENGTH,
-                #     interactive=True,
-                #     visible=True,
-                #     sources=[],
+                # fix_example = gr.Examples(
+                #     fix_example_imgs,
+                #     inputs=[fix_crop],
+                #     examples_per_page=20,
                 # )
-                fix_example = gr.Examples(
-                    fix_example_imgs,
-                    inputs=[fix_crop],
-                    examples_per_page=20,
-                    # run_on_click=True,
-                    # fn=load_brush,
-                )
             with gr.Column():
                 gr.Markdown(
                     """<p style="text-align: center; font-size: 18px; font-weight: bold;">2. Brush wrong finger and its surrounding area</p>"""
@@ -1460,19 +1398,10 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                     container=False,
                     interactive=True,
                 )
-                fix_ex_brush = gr.Examples(
-                    fix_example_brush,
-                    inputs=[fix_ref],
-                    # outputs=[fix_ref],
-                    examples_per_page=20,
-                    # run_on_click=True,
-                    # fn=inpaint_from_example,
-                )
-                # gr.Markdown(
-                #     """<p style="text-align: center;">&#9314; Hit the \"Finish Cropping & Brushing\" button</p>"""
-                # )
-                # fix_finish_crop = gr.Button(
-                #     value="Finish Croping & Brushing", interactive=False
+                # fix_ex_brush = gr.Examples(
+                #     fix_example_brush,
+                #     inputs=[fix_ref],
+                #     examples_per_page=20,
                 # )
             
             # keypoint selection
@@ -1485,8 +1414,8 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 )
                 fix_kp_all = gr.Image(
                     type="numpy",
-                    # label="Keypoints",
-                    show_label=False,
+                    label="Target Hand Pose",
+                    show_label=True,
                     height=LENGTH,
                     width=LENGTH,
                     interactive=False,
@@ -1494,14 +1423,14 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                     sources=(),
                     image_mode="RGBA"
                 )
-                with gr.Accordion(open=True):
-                    fix_ex_kpts = gr.Examples(
-                        fix_example_kpts,
-                        inputs=[fix_kp_all, fix_cfg, fix_seed, fix_kpts_path],
-                        examples_per_page=20,
-                        postprocess=False,
-                        elem_id="kpts_examples"
-                    )
+                # with gr.Accordion(open=True):
+                #     fix_ex_kpts = gr.Examples(
+                #         fix_example_kpts,
+                #         inputs=[fix_kp_all, fix_cfg, fix_seed, fix_kpts_path],
+                #         examples_per_page=20,
+                #         postprocess=False,
+                #         elem_id="kpts_examples"
+                #     )
                 with gr.Accordion("[Custom data] Manually give hand pose", open=False):
                     gr.Markdown(
                         """<p style="text-align: center;">&#9312; Tell us if this is right, left, or both hands</p>"""
@@ -1515,10 +1444,6 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                         """<p style="text-align: center;">&#9313; Click 21 keypoints on the image to provide the target hand pose of <b>right hand</b>. See the \"OpenPose keypoints convention\" for guidance.</p>""",
                         visible=False
                     )
-                    # fix_kp_r_info = gr.Markdown(
-                        # """<p style="text-align: center; font-size: 20px; font-weight: bold; ">Select right only</p>""",
-                        # visible=False,
-                    # )
                     fix_kp_right = gr.Image(
                         type="numpy",
                         label="Keypoint Selection (right hand)",
@@ -1569,21 +1494,11 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                         interactive=False,
                     )
             
-            # get latent
-            # with gr.Column():
-            
             # result column
             with gr.Column():
                 gr.Markdown(
                     """<p style="text-align: center; font-size: 18px; font-weight: bold;">4. Press &quot;Run&quot; to get the corrected hand image 🎯</p>"""
                 )
-                # gr.Markdown(
-                #     """<p style="text-align: center; font-size: 18px; font-weight: bold;">3. Press &quot;Ready&quot; to start pre-processing</p>"""
-                # )
-                # fix_ready = gr.Button(value="Ready", interactive=False)
-                # gr.Markdown(
-                #     """<p style="text-align: center; font-weight: bold; ">Visualized (256, 256)-resized, brushed image</p>"""
-                # )
                 fix_vis_mask32 = gr.Image(
                     type="numpy",
                     label=f"Visualized {opts.latent_size} Inpaint Mask",
@@ -1603,9 +1518,6 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                         interactive=False,
                         visible=True,
                     )
-                # gr.Markdown(
-                #     """<p style="text-align: center;">[NOTE] Above should be inpaint mask that you brushed, NOT the segmentation mask of the entire hand. </p>"""
-                # )
                 gr.Markdown(
                     """<p style="text-align: center;">⚠️  >3min and ~24GB per generation</p>"""
                 )
@@ -1645,7 +1557,14 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
                 )
                 fix_clear = gr.ClearButton()
 
-        
+            gr.Examples(
+                fix_example_all,
+                inputs=[fix_crop, fix_ref, fix_kp_all, fix_cfg, fix_seed, fix_kpts_path],
+                examples_per_page=20,
+                postprocess=False,
+                elem_id="fix_examples_all",
+            )
+
         # listeners
         fix_crop.change(stash_original, fix_crop, fix_original)  # fix_original: (real_H, real_W, 3)
         fix_crop.change(stay_crop, [fix_crop, fix_crop_coord], [fix_crop_coord, fix_ref])
@@ -1713,6 +1632,8 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
             reset_kps, [fix_img, fix_kpts, gr.State("left")], [fix_kp_left, fix_kpts]
         )
         fix_kpts_path.change(read_kpts, fix_kpts_path, fix_kpts_np)
+        fix_inpaint_mask.change(enable_component, [fix_inpaint_mask, fix_kpts_np], fix_run)
+        fix_kpts_np.change(enable_component, [fix_inpaint_mask, fix_kpts_np], fix_run)
         fix_run.click(
             ready_sample,
             [fix_ref, fix_inpaint_mask, fix_kpts, fix_kpts_np],
@@ -1820,13 +1741,14 @@ with gr.Blocks(css=custom_css, theme="soft") as demo:
             gr.Markdown("""<p style="text-align: center; font-size: 20px; font-weight: bold;">Tutorial Videos of Demo 2</p>""")
             with gr.Row(variant="panel", elem_id="repose_tutorial"):
                 with gr.Column():
-                    gr.Video(
-                        "how_to_videos/subtitled_repose_hands.mp4",
-                        label="Tutorial",
-                        autoplay=True,
-                        loop=True,
-                        show_label=True,
-                    )
+                    # gr.Video(
+                    #     "how_to_videos/subtitled_repose_hands.mp4",
+                    #     label="Tutorial",
+                    #     autoplay=True,
+                    #     loop=True,
+                    #     show_label=True,
+                    # )
+                    gr.HTML(tut2_example)
 
         # main tabs
         with gr.Row():