[{"paper":{"id":"2412.03558","authors":[{"_id":"67511976753718a53ff7fdfb","user":{"_id":"6375d136dee28348a9c63cbf","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6375d136dee28348a9c63cbf/gK465HBrQWIOZ-qtHb-Vh.jpeg","isPro":false,"fullname":"zehuan-huang","user":"huanngzh","type":"user","name":"huanngzh"},"name":"Zehuan Huang","status":"claimed_verified","statusLastChangedAt":"2024-12-05T09:18:15.519Z","hidden":false},{"_id":"67511976753718a53ff7fdfc","user":{"_id":"6346aaa3f06b237ba4e297b0","avatarUrl":"/avatars/5acb986e993eab1461200f3e9d99d022.svg","isPro":false,"fullname":"Yuan-Chen Guo","user":"bennyguo","type":"user","name":"bennyguo"},"name":"Yuan-Chen Guo","status":"admin_assigned","statusLastChangedAt":"2024-12-05T10:45:34.099Z","hidden":false},{"_id":"67511976753718a53ff7fdfd","user":{"_id":"674d7d6fe3908280620d1ac4","avatarUrl":"/avatars/29157f5a208937655c3f5e5f2ac23c6a.svg","isPro":false,"fullname":"Xingqiao_An","user":"XingqiaoAn","type":"user","name":"XingqiaoAn"},"name":"Xingqiao An","status":"admin_assigned","statusLastChangedAt":"2024-12-05T10:45:39.396Z","hidden":false},{"_id":"67511976753718a53ff7fdfe","user":{"_id":"64b4eecf2fc8324fcb63b404","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64b4eecf2fc8324fcb63b404/zGYqYVB4-o-GBMybJ8CDA.png","isPro":false,"fullname":"Yunhan Yang","user":"yhyang-myron","type":"user","name":"yhyang-myron"},"name":"Yunhan Yang","status":"admin_assigned","statusLastChangedAt":"2024-12-05T10:45:45.545Z","hidden":false},{"_id":"67511976753718a53ff7fdff","user":{"_id":"64d71083a787c9bc7b9f1238","avatarUrl":"/avatars/d0b0546dec7fc5792921154bec41385a.svg","isPro":false,"fullname":"Yangguang Li","user":"Lp256","type":"user","name":"Lp256"},"name":"Yangguang Li","status":"claimed_verified","statusLastChangedAt":"2025-02-14T16:20:16.593Z","hidden":false},{"_id":"67511976753718a53ff7fe00","user":{"_id":"644dbf6453ad80c6593bf748","avatarUrl":"/avatars/0e170cf2aa8d7f0f3f83e36f06f023f8.svg","isPro":false,"fullname":"Zixin Zou","user":"zouzx","type":"user","name":"zouzx"},"name":"Zi-Xin Zou","status":"admin_assigned","statusLastChangedAt":"2024-12-05T10:46:03.490Z","hidden":false},{"_id":"67511976753718a53ff7fe01","name":"Ding Liang","hidden":false},{"_id":"67511976753718a53ff7fe02","user":{"_id":"65d5ec74cd05bc1eaa125040","avatarUrl":"/avatars/2de1b1539a86452c2c89570eeb02f5ab.svg","isPro":false,"fullname":"Xihui Liu","user":"XihuiLiu","type":"user","name":"XihuiLiu"},"name":"Xihui Liu","status":"admin_assigned","statusLastChangedAt":"2024-12-05T10:46:12.847Z","hidden":false},{"_id":"67511976753718a53ff7fe03","user":{"_id":"638066faf022c8a5803f7eb8","avatarUrl":"/avatars/4cfd699c3f6c5461b12b7dc5e3fe183d.svg","isPro":false,"fullname":"Yanpei Cao","user":"pookiefoof","type":"user","name":"pookiefoof"},"name":"Yan-Pei Cao","status":"admin_assigned","statusLastChangedAt":"2024-12-05T10:46:20.698Z","hidden":false},{"_id":"67511976753718a53ff7fe04","user":{"_id":"65b722dbe02a17f0f8d1cc6b","avatarUrl":"/avatars/65f20601ef9b8ebfdddadd737f9153d6.svg","isPro":false,"fullname":"Lu Sheng","user":"lsheng2024","type":"user","name":"lsheng2024"},"name":"Lu Sheng","status":"claimed_verified","statusLastChangedAt":"2024-12-06T15:57:38.089Z","hidden":false}],"publishedAt":"2024-12-04T18:52:40.000Z","title":"MIDI: Multi-Instance Diffusion for Single Image to 3D Scene Generation","summary":"This paper introduces MIDI, a novel paradigm for compositional 3D scene\ngeneration from a single image. Unlike existing methods that rely on\nreconstruction or retrieval techniques or recent approaches that employ\nmulti-stage object-by-object generation, MIDI extends pre-trained image-to-3D\nobject generation models to multi-instance diffusion models, enabling the\nsimultaneous generation of multiple 3D instances with accurate spatial\nrelationships and high generalizability. At its core, MIDI incorporates a novel\nmulti-instance attention mechanism, that effectively captures inter-object\ninteractions and spatial coherence directly within the generation process,\nwithout the need for complex multi-step processes. The method utilizes partial\nobject images and global scene context as inputs, directly modeling object\ncompletion during 3D generation. During training, we effectively supervise the\ninteractions between 3D instances using a limited amount of scene-level data,\nwhile incorporating single-object data for regularization, thereby maintaining\nthe pre-trained generalization ability. MIDI demonstrates state-of-the-art\nperformance in image-to-scene generation, validated through evaluations on\nsynthetic data, real-world scene data, and stylized scene images generated by\ntext-to-image diffusion models.","upvotes":21,"discussionId":"67511978753718a53ff7fe9d","projectPage":"https://huanngzh.github.io/MIDI-Page/","githubRepo":"https://github.com/VAST-AI-Research/MIDI-3D","githubRepoAddedBy":"user","ai_summary":"MIDI is a novel method for 3D scene generation from a single image using multi-instance diffusion models with attention mechanisms, achieving state-of-the-art results across different datasets.","ai_keywords":["image-to-3D object generation models","diffusion models","multi-instance attention mechanism","partial object images","global scene context","scene-level data","single-object data"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":950},"publishedAt":"2024-12-04T13:52:40.000Z","title":"MIDI: Multi-Instance Diffusion for Single Image to 3D Scene Generation","summary":"This paper introduces MIDI, a novel paradigm for compositional 3D scene\ngeneration from a single image. Unlike existing methods that rely on\nreconstruction or retrieval techniques or recent approaches that employ\nmulti-stage object-by-object generation, MIDI extends pre-trained image-to-3D\nobject generation models to multi-instance diffusion models, enabling the\nsimultaneous generation of multiple 3D instances with accurate spatial\nrelationships and high generalizability. At its core, MIDI incorporates a novel\nmulti-instance attention mechanism, that effectively captures inter-object\ninteractions and spatial coherence directly within the generation process,\nwithout the need for complex multi-step processes. The method utilizes partial\nobject images and global scene context as inputs, directly modeling object\ncompletion during 3D generation. During training, we effectively supervise the\ninteractions between 3D instances using a limited amount of scene-level data,\nwhile incorporating single-object data for regularization, thereby maintaining\nthe pre-trained generalization ability. MIDI demonstrates state-of-the-art\nperformance in image-to-scene generation, validated through evaluations on\nsynthetic data, real-world scene data, and stylized scene images generated by\ntext-to-image diffusion models.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2412.03558.png","numComments":3,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"MIDI: "},{"type":"highlight","text":"Multi"},{"type":"text","text":"-"},{"type":"highlight","text":"Instance"},{"type":"text","text":" Diffusion for "},{"type":"highlight","text":"Single"},{"type":"text","text":" "},{"type":"highlight","text":"Image"},{"type":"text","text":" "},{"type":"highlight","text":"to"},{"type":"text","text":" "},{"type":"highlight","text":"3D"},{"type":"text","text":" "},{"type":"highlight","text":"Scene"},{"type":"text","text":" "},{"type":"highlight","text":"Generation"}],"highlightedSummary":[{"type":"text","text":"This paper introduces MIDI, a novel paradigm for "},{"type":"highlight","text":"compositional"},{"type":"text","text":" "},{"type":"highlight","text":"3D"},{"type":"text","text":" "},{"type":"highlight","text":"scene"},{"type":"text","text":"\n"},{"type":"highlight","text":"generation"},{"type":"text","text":" from a "},{"type":"highlight","text":"single"},{"type":"text","text":" "},{"type":"highlight","text":"image."},{"type":"text","text":" Unlike existing methods that rely on\nreconstruction or retrieval techniques or recent approaches that employ\nmulti-stage object-by-object generation, MIDI extends pre-trained image-to-3D\nobject generation models to multi-instance diffusion models, enabling the\nsimultaneous generation of multiple 3D instances with accurate spatial\nrelationships and high generalizability. At its core, MIDI incorporates a novel\nmulti-instance attention mechanism, that effectively captures inter-object\ninteractions and spatial coherence directly within the generation process,\nwithout the need for complex multi-step processes. The method utilizes partial\nobject images and global scene context as inputs, directly modeling object\ncompletion during 3D generation. During training, we effectively supervise the\ninteractions between 3D instances using a limited amount of scene-level data,\nwhile incorporating single-object data for regularization, thereby maintaining\nthe pre-trained generalization ability. MIDI demonstrates state-of-the-art\nperformance in image-to-scene generation, validated through evaluations on\nsynthetic data, real-world scene data, and stylized scene images generated by\ntext-to-image diffusion models."}]},{"paper":{"id":"2609.05416","authors":[{"_id":"6a9e1a56de5ea82090db6541","name":"Muyao Niu","hidden":false},{"_id":"6a9e1a56de5ea82090db6542","user":{"_id":"64a5bda1733731f38378fdd3","avatarUrl":"/avatars/920d88b58752b6e9ce759887fe8a0bdd.svg","isPro":false,"fullname":"Jixuan HE","user":"Kakituken","type":"user","name":"Kakituken"},"name":"Jixuan He","status":"claimed_verified","statusLastChangedAt":"2026-09-10T08:45:05.116Z","hidden":false},{"_id":"6a9e1a56de5ea82090db6543","name":"Ruihan Yu","hidden":false},{"_id":"6a9e1a56de5ea82090db6544","name":"Lian Fu","hidden":false},{"_id":"6a9e1a56de5ea82090db6545","name":"Yonghao Yu","hidden":false},{"_id":"6a9e1a56de5ea82090db6546","name":"Zheng-Hui Huang","hidden":false},{"_id":"6a9e1a56de5ea82090db6547","name":"Yifan Zhan","hidden":false},{"_id":"6a9e1a56de5ea82090db6548","name":"Fengbo Lan","hidden":false},{"_id":"6a9e1a56de5ea82090db6549","name":"Yongtao Ge","hidden":false},{"_id":"6a9e1a56de5ea82090db654a","name":"Yinqiang Zheng","hidden":false},{"_id":"6a9e1a56de5ea82090db654b","name":"Kaipeng Zhang","hidden":false},{"_id":"6a9e1a56de5ea82090db654c","user":{"_id":"6307a98795b2ab342fec0cf7","avatarUrl":"/avatars/85b261bcdda4717a6e40491f6c7b7a89.svg","isPro":false,"fullname":"Zhixiang Wang","user":"wangzx1994","type":"user","name":"wangzx1994"},"name":"Zhixiang Wang","status":"claimed_verified","statusLastChangedAt":"2026-09-07T16:45:04.043Z","hidden":false}],"publishedAt":"2026-09-04T00:00:00.000Z","title":"WorldSculpt: Generating Compositional Worlds from Grounded Videos","summary":"We study the problem of generating a compositional 3D representation of a cluttered scene containing hundreds of objects. The goal is to represent the scene as a collection of individual object meshes placed in a shared world frame, as required by downstream applications such as gaming, AR/VR, simulation, and robotics. This task is challenging in densely cluttered scenes, where objects heavily occlude one another and each view reveals only a fraction of their geometry. Geometry-based approaches typically reconstruct the scene as a single representation and leave incomplete geometry in occluded regions, while existing compositional methods with generative priors are largely limited to relatively simple scenes. We show that complex scenes with hundreds of objects can instead be generated compositionally by adapting a strong single-object 3D generative prior to multi-view observations. We instantiate this paradigm with Pixal3D, extending it with a multi-view conditioning pathway that grounds object generation in multiple posed observations. Although the model is finetuned entirely on single objects in canonical space, it generalizes to large scenes with severe occlusion without any scene-level training, demonstrating the feasibility and scalability of this paradigm. We further introduce UE-MeshyScene, a photorealistic benchmark of densely cluttered scenes with hundreds of objects, per-object annotations, and ground-truth meshes. Across single-object, controlled multi-object, and UE-MeshyScene evaluations, our method consistently outperforms prior approaches, with larger gains as scene complexity and occlusion increase. Finally, we demonstrate broader applicability by converting generated 3DGS worlds, such as Marble and HY-World 2.0, into compositional mesh scenes.","upvotes":25,"discussionId":"6a9e1a57de5ea82090db654d","projectPage":"https://alaya-lab.github.io/WorldSculpt/","githubRepo":"https://github.com/AlayaLab/WorldSculpt","githubRepoAddedBy":"user","ai_summary":"Adapting a single-object 3D generative prior to multi-view observations enables scalable compositional mesh reconstruction of densely cluttered scenes with severe occlusion.","ai_keywords":["compositional 3D representation","object meshes","multi-view conditioning","3D generative prior","Pixal3D","3DGS","UE-MeshyScene"],"ai_summary_model":"thinkingmachines/Inkling-Small","githubStars":325},"publishedAt":"2026-09-03T20:00:00.000Z","title":"WorldSculpt: Generating Compositional Worlds from Grounded Videos","summary":"We study the problem of generating a compositional 3D representation of a cluttered scene containing hundreds of objects. The goal is to represent the scene as a collection of individual object meshes placed in a shared world frame, as required by downstream applications such as gaming, AR/VR, simulation, and robotics. This task is challenging in densely cluttered scenes, where objects heavily occlude one another and each view reveals only a fraction of their geometry. Geometry-based approaches typically reconstruct the scene as a single representation and leave incomplete geometry in occluded regions, while existing compositional methods with generative priors are largely limited to relatively simple scenes. We show that complex scenes with hundreds of objects can instead be generated compositionally by adapting a strong single-object 3D generative prior to multi-view observations. We instantiate this paradigm with Pixal3D, extending it with a multi-view conditioning pathway that grounds object generation in multiple posed observations. Although the model is finetuned entirely on single objects in canonical space, it generalizes to large scenes with severe occlusion without any scene-level training, demonstrating the feasibility and scalability of this paradigm. We further introduce UE-MeshyScene, a photorealistic benchmark of densely cluttered scenes with hundreds of objects, per-object annotations, and ground-truth meshes. Across single-object, controlled multi-object, and UE-MeshyScene evaluations, our method consistently outperforms prior approaches, with larger gains as scene complexity and occlusion increase. Finally, we demonstrate broader applicability by converting generated 3DGS worlds, such as Marble and HY-World 2.0, into compositional mesh scenes.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2609.05416.png","numComments":1,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"WorldSculpt: Generating Compositional Worlds from Grounded Videos"}],"highlightedSummary":[{"type":"text","text":"We study the problem of generating a compositional 3D representation of a cluttered scene containing hundreds of objects. The goal is to represent the scene as a collection of individual object meshes placed in a shared world frame, as required by downstream applications such as gaming, AR/VR, simulation, and robotics. This task is challenging in densely cluttered scenes, where objects heavily occlude one another and each view reveals only a fraction of their geometry. Geometry-based approaches typically reconstruct the scene as a single representation and leave incomplete geometry in occluded regions, while existing compositional methods with generative priors are largely limited to relatively simple scenes. "},{"type":"highlight","text":"We show that complex scenes with hundreds of objects can instead be generated compositionally by adapting a strong single-object 3D generative prior to multi-view observations."},{"type":"text","text":" We instantiate this paradigm with Pixal3D, extending it with a multi-view conditioning pathway that grounds object generation in multiple posed observations. Although the model is finetuned entirely on single objects in canonical space, it generalizes to large scenes with severe occlusion without any scene-level training, demonstrating the feasibility and scalability of this paradigm. We further introduce UE-MeshyScene, a photorealistic benchmark of densely cluttered scenes with hundreds of objects, per-object annotations, and ground-truth meshes. Across single-object, controlled multi-object, and UE-MeshyScene evaluations, our method consistently outperforms prior approaches, with larger gains as scene complexity and occlusion increase. Finally, we demonstrate broader applicability by converting generated 3DGS worlds, such as Marble and HY-World 2.0, into compositional mesh scenes."}]},{"paper":{"id":"2604.04406","authors":[{"_id":"69d5d7eeb9b236b710ae81f8","name":"Ze-Xin Yin","hidden":false},{"_id":"69d5d7eeb9b236b710ae81f9","user":{"_id":"6534ca673da0ff3c709b5852","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6534ca673da0ff3c709b5852/mIxuLmc_ntALsG83ZLfBh.jpeg","isPro":false,"fullname":"Liu Liu","user":"nemo04","type":"user","name":"nemo04"},"name":"Liu Liu","status":"claimed_verified","statusLastChangedAt":"2026-09-02T16:45:04.668Z","hidden":false},{"_id":"69d5d7eeb9b236b710ae81fa","name":"Xinjie Wang","hidden":false},{"_id":"69d5d7eeb9b236b710ae81fb","name":"Wei Sui","hidden":false},{"_id":"69d5d7eeb9b236b710ae81fc","name":"Zhizhong Su","hidden":false},{"_id":"69d5d7eeb9b236b710ae81fd","name":"Jian Yang","hidden":false},{"_id":"69d5d7eeb9b236b710ae81fe","name":"Jin Xie","hidden":false}],"publishedAt":"2026-04-06T00:00:00.000Z","title":"3D-Fixer: Coarse-to-Fine In-place Completion for 3D Scenes from a Single Image","summary":"Compositional 3D scene generation from a single view requires the simultaneous recovery of scene layout and 3D assets. Existing approaches mainly fall into two categories: feed-forward generation methods and per-instance generation methods. The former directly predict 3D assets with explicit 6DoF poses through efficient network inference, but they generalize poorly to complex scenes. The latter improve generalization through a divide-and-conquer strategy, but suffer from time-consuming pose optimization. To bridge this gap, we introduce 3D-Fixer, a novel in-place completion paradigm. Specifically, 3D-Fixer extends 3D object generative priors to generate complete 3D assets conditioned on the partially visible point cloud at the original locations, which are cropped from the fragmented geometry obtained from the geometry estimation methods. Unlike prior works that require explicit pose alignment, 3D-Fixer uses fragmented geometry as a spatial anchor to preserve layout fidelity. At its core, we propose a coarse-to-fine generation scheme to resolve boundary ambiguity under occlusion, supported by a dual-branch conditioning network and an Occlusion-Robust Feature Alignment (ORFA) strategy for stable training. Furthermore, to address the data scarcity bottleneck, we present ARSG-110K, the largest scene-level dataset to date, comprising over 110K diverse scenes and 3M annotated images with high-fidelity 3D ground truth. Extensive experiments show that 3D-Fixer achieves state-of-the-art geometric accuracy, which significantly outperforms baselines such as MIDI and Gen3DSR, while maintaining the efficiency of the diffusion process. Code and data will be publicly available at https://zx-yin.github.io/3dfixer.","upvotes":0,"discussionId":"69d5d7eeb9b236b710ae81ff","ai_summary":"3D-Fixer enables efficient and accurate 3D scene generation from single views by using fragmented geometry as spatial anchors and a coarse-to-fine generation approach that handles occlusions and boundary ambiguities.","ai_keywords":["3D scene generation","3D object generative priors","point cloud","fragmented geometry","coarse-to-fine generation","dual-branch conditioning network","Occlusion-Robust Feature Alignment","diffusion process","3D asset generation","scene layout recovery"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2026-04-05T20:00:00.000Z","title":"3D-Fixer: Coarse-to-Fine In-place Completion for 3D Scenes from a Single Image","summary":"Compositional 3D scene generation from a single view requires the simultaneous recovery of scene layout and 3D assets. Existing approaches mainly fall into two categories: feed-forward generation methods and per-instance generation methods. The former directly predict 3D assets with explicit 6DoF poses through efficient network inference, but they generalize poorly to complex scenes. The latter improve generalization through a divide-and-conquer strategy, but suffer from time-consuming pose optimization. To bridge this gap, we introduce 3D-Fixer, a novel in-place completion paradigm. Specifically, 3D-Fixer extends 3D object generative priors to generate complete 3D assets conditioned on the partially visible point cloud at the original locations, which are cropped from the fragmented geometry obtained from the geometry estimation methods. Unlike prior works that require explicit pose alignment, 3D-Fixer uses fragmented geometry as a spatial anchor to preserve layout fidelity. At its core, we propose a coarse-to-fine generation scheme to resolve boundary ambiguity under occlusion, supported by a dual-branch conditioning network and an Occlusion-Robust Feature Alignment (ORFA) strategy for stable training. Furthermore, to address the data scarcity bottleneck, we present ARSG-110K, the largest scene-level dataset to date, comprising over 110K diverse scenes and 3M annotated images with high-fidelity 3D ground truth. Extensive experiments show that 3D-Fixer achieves state-of-the-art geometric accuracy, which significantly outperforms baselines such as MIDI and Gen3DSR, while maintaining the efficiency of the diffusion process. Code and data will be publicly available at https://zx-yin.github.io/3dfixer.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2604.04406.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"3D-Fixer: Coarse-to-Fine In-place Completion for 3D Scenes from a Single Image"}],"highlightedSummary":[{"type":"highlight","text":"Compositional 3D scene generation from a single view requires the simultaneous recovery of scene layout and 3D assets."},{"type":"text","text":" Existing approaches mainly fall into two categories: feed-forward generation methods and per-instance generation methods. The former directly predict 3D assets with explicit 6DoF poses through efficient network inference, but they generalize poorly to complex scenes. The latter improve generalization through a divide-and-conquer strategy, but suffer from time-consuming pose optimization. To bridge this gap, we introduce 3D-Fixer, a novel in-place completion paradigm. Specifically, 3D-Fixer extends 3D object generative priors to generate complete 3D assets conditioned on the partially visible point cloud at the original locations, which are cropped from the fragmented geometry obtained from the geometry estimation methods. Unlike prior works that require explicit pose alignment, 3D-Fixer uses fragmented geometry as a spatial anchor to preserve layout fidelity. At its core, we propose a coarse-to-fine generation scheme to resolve boundary ambiguity under occlusion, supported by a dual-branch conditioning network and an Occlusion-Robust Feature Alignment (ORFA) strategy for stable training. Furthermore, to address the data scarcity bottleneck, we present ARSG-110K, the largest scene-level dataset to date, comprising over 110K diverse scenes and 3M annotated images with high-fidelity 3D ground truth. Extensive experiments show that 3D-Fixer achieves state-of-the-art geometric accuracy, which significantly outperforms baselines such as MIDI and Gen3DSR, while maintaining the efficiency of the diffusion process. Code and data will be publicly available at https://zx-yin.github.io/3dfixer."}]},{"paper":{"id":"2011.12100","authors":[{"_id":"6504554657cc1e5d753770a4","name":"Michael Niemeyer","hidden":false},{"_id":"6504554657cc1e5d753770a5","name":"Andreas Geiger","hidden":false}],"publishedAt":"2020-11-24T14:14:15.000Z","title":"GIRAFFE: Representing Scenes as Compositional Generative Neural Feature\n  Fields","summary":"Deep generative models allow for photorealistic image synthesis at high\nresolutions. But for many applications, this is not enough: content creation\nalso needs to be controllable. While several recent works investigate how to\ndisentangle underlying factors of variation in the data, most of them operate\nin 2D and hence ignore that our world is three-dimensional. Further, only few\nworks consider the compositional nature of scenes. Our key hypothesis is that\nincorporating a compositional 3D scene representation into the generative model\nleads to more controllable image synthesis. Representing scenes as\ncompositional generative neural feature fields allows us to disentangle one or\nmultiple objects from the background as well as individual objects' shapes and\nappearances while learning from unstructured and unposed image collections\nwithout any additional supervision. Combining this scene representation with a\nneural rendering pipeline yields a fast and realistic image synthesis model. As\nevidenced by our experiments, our model is able to disentangle individual\nobjects and allows for translating and rotating them in the scene as well as\nchanging the camera pose.","upvotes":0,"discussionId":"6504554757cc1e5d753770ff","ai_summary":"Integrating compositional 3D scene representation in deep generative models enables disentangled image synthesis with controllable objects and camera poses from unstructured image data.","ai_keywords":["deep generative models","photorealistic image synthesis","compositional 3D scene representation","generative neural feature fields","disentanglement","neural rendering pipeline","image synthesis","camera pose"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2020-11-24T09:14:15.000Z","title":"GIRAFFE: Representing Scenes as Compositional Generative Neural Feature\n  Fields","summary":"Deep generative models allow for photorealistic image synthesis at high\nresolutions. But for many applications, this is not enough: content creation\nalso needs to be controllable. While several recent works investigate how to\ndisentangle underlying factors of variation in the data, most of them operate\nin 2D and hence ignore that our world is three-dimensional. Further, only few\nworks consider the compositional nature of scenes. Our key hypothesis is that\nincorporating a compositional 3D scene representation into the generative model\nleads to more controllable image synthesis. Representing scenes as\ncompositional generative neural feature fields allows us to disentangle one or\nmultiple objects from the background as well as individual objects' shapes and\nappearances while learning from unstructured and unposed image collections\nwithout any additional supervision. Combining this scene representation with a\nneural rendering pipeline yields a fast and realistic image synthesis model. As\nevidenced by our experiments, our model is able to disentangle individual\nobjects and allows for translating and rotating them in the scene as well as\nchanging the camera pose.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2011.12100.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"GIRAFFE: Representing Scenes as Compositional Generative Neural Feature\n  Fields"}],"highlightedSummary":[{"type":"text","text":"Deep generative models allow for photorealistic image synthesis at high\nresolutions. But for many applications, this is not enough: content creation\nalso needs to be controllable. While several recent works investigate how to\ndisentangle underlying factors of variation in the data, most of them operate\nin 2D and hence ignore that our world is three-dimensional. "},{"type":"highlight","text":"Further, only few\nworks consider the compositional nature of scenes."},{"type":"text","text":" Our key hypothesis is that\nincorporating a compositional 3D scene representation into the generative model\nleads to more controllable image synthesis. Representing scenes as\ncompositional generative neural feature fields allows us to disentangle one or\nmultiple objects from the background as well as individual objects' shapes and\nappearances while learning from unstructured and unposed image collections\nwithout any additional supervision. Combining this scene representation with a\nneural rendering pipeline yields a fast and realistic image synthesis model. As\nevidenced by our experiments, our model is able to disentangle individual\nobjects and allows for translating and rotating them in the scene as well as\nchanging the camera pose."}]},{"paper":{"id":"2303.12074","authors":[{"_id":"64f463502407dbe79fe2a229","name":"Sherwin Bahmani","hidden":false},{"_id":"64f463502407dbe79fe2a22a","user":{"_id":"6517d7f84fadbeb643c68269","avatarUrl":"/avatars/ad21288123abcd4488ae1bdcb10b54e7.svg","isPro":false,"fullname":"JJ Park","user":"jjparkcv","type":"user","name":"jjparkcv"},"name":"Jeong Joon Park","status":"claimed_verified","statusLastChangedAt":"2024-07-11T17:35:00.200Z","hidden":false},{"_id":"64f463502407dbe79fe2a22b","name":"Despoina Paschalidou","hidden":false},{"_id":"64f463502407dbe79fe2a22c","name":"Xingguang Yan","hidden":false},{"_id":"64f463502407dbe79fe2a22d","name":"Gordon Wetzstein","hidden":false},{"_id":"64f463502407dbe79fe2a22e","name":"Leonidas Guibas","hidden":false},{"_id":"64f463502407dbe79fe2a22f","name":"Andrea Tagliasacchi","hidden":false}],"publishedAt":"2023-03-21T17:59:02.000Z","title":"CC3D: Layout-Conditioned Generation of Compositional 3D Scenes","summary":"In this work, we introduce CC3D, a conditional generative model that\nsynthesizes complex 3D scenes conditioned on 2D semantic scene layouts, trained\nusing single-view images. Different from most existing 3D GANs that limit their\napplicability to aligned single objects, we focus on generating complex scenes\nwith multiple objects, by modeling the compositional nature of 3D scenes. By\ndevising a 2D layout-based approach for 3D synthesis and implementing a new 3D\nfield representation with a stronger geometric inductive bias, we have created\na 3D GAN that is both efficient and of high quality, while allowing for a more\ncontrollable generation process. Our evaluations on synthetic 3D-FRONT and\nreal-world KITTI-360 datasets demonstrate that our model generates scenes of\nimproved visual and geometric quality in comparison to previous works.","upvotes":1,"discussionId":"64f463522407dbe79fe2a27a","ai_summary":"A 3D generative adversarial network (GAN) synthesized from 2D layouts generates high-quality, complex 3D scenes, outperforming earlier models in visual and geometric quality.","ai_keywords":["conditional generative model","3D GANs","single-view images","2D semantic scene layouts","compositional nature","3D field representation","geometric inductive bias","3D-FRONT","KITTI-360"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2023-03-21T13:59:02.000Z","title":"CC3D: Layout-Conditioned Generation of Compositional 3D Scenes","summary":"In this work, we introduce CC3D, a conditional generative model that\nsynthesizes complex 3D scenes conditioned on 2D semantic scene layouts, trained\nusing single-view images. Different from most existing 3D GANs that limit their\napplicability to aligned single objects, we focus on generating complex scenes\nwith multiple objects, by modeling the compositional nature of 3D scenes. By\ndevising a 2D layout-based approach for 3D synthesis and implementing a new 3D\nfield representation with a stronger geometric inductive bias, we have created\na 3D GAN that is both efficient and of high quality, while allowing for a more\ncontrollable generation process. Our evaluations on synthetic 3D-FRONT and\nreal-world KITTI-360 datasets demonstrate that our model generates scenes of\nimproved visual and geometric quality in comparison to previous works.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2303.12074.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"CC3D: Layout-Conditioned Generation of Compositional 3D Scenes"}],"highlightedSummary":[{"type":"text","text":"In this work, we introduce CC3D, a conditional generative model that\nsynthesizes complex 3D scenes conditioned on 2D semantic scene layouts, trained\nusing single-view images. "},{"type":"highlight","text":"Different from most existing 3D GANs that limit their\napplicability to aligned single objects, we focus on generating complex scenes\nwith multiple objects, by modeling the compositional nature of 3D scenes."},{"type":"text","text":" By\ndevising a 2D layout-based approach for 3D synthesis and implementing a new 3D\nfield representation with a stronger geometric inductive bias, we have created\na 3D GAN that is both efficient and of high quality, while allowing for a more\ncontrollable generation process. Our evaluations on synthetic 3D-FRONT and\nreal-world KITTI-360 datasets demonstrate that our model generates scenes of\nimproved visual and geometric quality in comparison to previous works."}]},{"paper":{"id":"2405.18525","authors":[{"_id":"69c60bd52de2ee20094edc3f","user":{"_id":"66867085636d8845cd498b98","avatarUrl":"/avatars/d9cabeeca3f0cffff3841c2df3e88c5e.svg","isPro":false,"fullname":"Vincent Han","user":"VincentHancoder","type":"user","name":"VincentHancoder"},"name":"Haonan Han","status":"claimed_verified","statusLastChangedAt":"2026-03-27T11:33:37.172Z","hidden":true},{"_id":"69c60bd52de2ee20094edc40","name":"Rui Yang","hidden":false},{"_id":"69c60bd52de2ee20094edc41","name":"Huan Liao","hidden":false},{"_id":"69c60bd52de2ee20094edc42","name":"Jiankai Xing","hidden":false},{"_id":"69c60bd52de2ee20094edc43","name":"Zunnan Xu","hidden":false},{"_id":"69c60bd52de2ee20094edc44","name":"Xiaoming Yu","hidden":false},{"_id":"69c60bd52de2ee20094edc45","name":"Junwei Zha","hidden":false},{"_id":"69c60bd52de2ee20094edc46","name":"Xiu Li","hidden":false},{"_id":"69c60bd52de2ee20094edc47","name":"Wanhua Li","hidden":false}],"publishedAt":"2024-05-28T18:45:10.000Z","title":"REPARO: Compositional 3D Assets Generation with Differentiable 3D Layout Alignment","summary":"Traditional image-to-3D models often struggle with scenes containing multiple objects due to biases and occlusion complexities. To address this challenge, we present REPARO, a novel approach for compositional 3D asset generation from single images. REPARO employs a two-step process: first, it extracts individual objects from the scene and reconstructs their 3D meshes using off-the-shelf image-to-3D models; then, it optimizes the layout of these meshes through differentiable rendering techniques, ensuring coherent scene composition. By integrating optimal transport-based long-range appearance loss term and high-level semantic loss term in the differentiable rendering, REPARO can effectively recover the layout of 3D assets. The proposed method can significantly enhance object independence, detail accuracy, and overall scene coherence. Extensive evaluation of multi-object scenes demonstrates that our REPARO offers a comprehensive approach to address the complexities of multi-object 3D scene generation from single images.","upvotes":0,"discussionId":"69c60bd52de2ee20094edc48","ai_summary":"REPARO generates compositional 3D scenes from single images by extracting individual objects, reconstructing their 3D meshes, and optimizing scene layout through differentiable rendering with appearance and semantic loss terms.","ai_keywords":["image-to-3D models","3D mesh reconstruction","differentiable rendering","optimal transport","appearance loss","semantic loss","scene composition","object independence","detail accuracy","scene coherence"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-05-28T14:45:10.000Z","title":"REPARO: Compositional 3D Assets Generation with Differentiable 3D Layout Alignment","summary":"Traditional image-to-3D models often struggle with scenes containing multiple objects due to biases and occlusion complexities. To address this challenge, we present REPARO, a novel approach for compositional 3D asset generation from single images. REPARO employs a two-step process: first, it extracts individual objects from the scene and reconstructs their 3D meshes using off-the-shelf image-to-3D models; then, it optimizes the layout of these meshes through differentiable rendering techniques, ensuring coherent scene composition. By integrating optimal transport-based long-range appearance loss term and high-level semantic loss term in the differentiable rendering, REPARO can effectively recover the layout of 3D assets. The proposed method can significantly enhance object independence, detail accuracy, and overall scene coherence. Extensive evaluation of multi-object scenes demonstrates that our REPARO offers a comprehensive approach to address the complexities of multi-object 3D scene generation from single images.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2405.18525.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"REPARO: Compositional 3D Assets Generation with Differentiable 3D Layout Alignment"}],"highlightedSummary":[{"type":"text","text":"Traditional image-to-3D models often struggle with scenes containing multiple objects due to biases and occlusion complexities. "},{"type":"highlight","text":"To address this challenge, we present REPARO, a novel approach for compositional 3D asset generation from single images."},{"type":"text","text":" REPARO employs a two-step process: first, it extracts individual objects from the scene and reconstructs their 3D meshes using off-the-shelf image-to-3D models; then, it optimizes the layout of these meshes through differentiable rendering techniques, ensuring coherent scene composition. By integrating optimal transport-based long-range appearance loss term and high-level semantic loss term in the differentiable rendering, REPARO can effectively recover the layout of 3D assets. The proposed method can significantly enhance object independence, detail accuracy, and overall scene coherence. Extensive evaluation of multi-object scenes demonstrates that our REPARO offers a comprehensive approach to address the complexities of multi-object 3D scene generation from single images."}]},{"paper":{"id":"2603.02133","authors":[{"_id":"69b76f4691a28162b233fc12","user":{"_id":"676c04f44464f476aaa53d1c","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/no-auth/k488J1893F3JGwEMvaeuh.png","isPro":false,"fullname":"Chong Xia","user":"xiac24","type":"user","name":"xiac24"},"name":"Chong Xia","status":"admin_assigned","statusLastChangedAt":"2026-03-16T11:13:44.499Z","hidden":false},{"_id":"69b76f4691a28162b233fc13","name":"Kai Zhu","hidden":false},{"_id":"69b76f4691a28162b233fc14","user":{"_id":"690eb31aeed4fe1e4fa0daf7","avatarUrl":"/avatars/3ef17cb031fc9ce5fa1d7d43fbd25d5a.svg","isPro":false,"fullname":"Zizhuo Wang","user":"Wzz234","type":"user","name":"Wzz234"},"name":"Zizhuo Wang","status":"admin_assigned","statusLastChangedAt":"2026-03-16T11:13:55.765Z","hidden":false},{"_id":"69b76f4691a28162b233fc15","user":{"_id":"6505a02f9310ce8c400edc63","avatarUrl":"/avatars/bbf781594fc8c812316711aa8e2797aa.svg","isPro":false,"fullname":"Fangfu Liu","user":"Liuff23","type":"user","name":"Liuff23"},"name":"Fangfu Liu","status":"admin_assigned","statusLastChangedAt":"2026-03-16T11:14:01.951Z","hidden":false},{"_id":"69b76f4691a28162b233fc16","name":"Zhizheng Zhang","hidden":false},{"_id":"69b76f4691a28162b233fc17","user":{"_id":"66c8131afafc0fc87ca99650","avatarUrl":"/avatars/a6eeba2ccf011d5c9964fd38f85bd671.svg","isPro":false,"fullname":"Yueqi Duan","user":"duanyueqi","type":"user","name":"duanyueqi"},"name":"Yueqi Duan","status":"admin_assigned","statusLastChangedAt":"2026-03-16T11:14:11.512Z","hidden":false}],"publishedAt":"2026-03-02T17:51:45.000Z","title":"SimRecon: SimReady Compositional Scene Reconstruction from Real Videos","summary":"Compositional scene reconstruction seeks to create object-centric representations rather than holistic scenes from real-world videos, which is natively applicable for simulation and interaction. Conventional compositional reconstruction approaches primarily emphasize on visual appearance and show limited generalization ability to real-world scenarios. In this paper, we propose SimRecon, a framework that realizes a \"Perception-Generation-Simulation\" pipeline towards cluttered scene reconstruction, which first conducts scene-level semantic reconstruction from video input, then performs single-object generation, and finally assembles these assets in the simulator. However, naively combining these three stages leads to visual infidelity of generated assets and physical implausibility of the final scene, a problem particularly severe for complex scenes. Thus, we further propose two bridging modules between the three stages to address this problem. To be specific, for the transition from Perception to Generation, critical for visual fidelity, we introduce Active Viewpoint Optimization, which actively searches in 3D space to acquire optimal projected images as conditions for single-object completion. Moreover, for the transition from Generation to Simulation, essential for physical plausibility, we propose a Scene Graph Synthesizer, which guides the construction from scratch in 3D simulators, mirroring the native, constructive principle of the real world. Extensive experiments on the ScanNet dataset validate our method's superior performance over previous state-of-the-art approaches.","upvotes":4,"discussionId":"69b76f4691a28162b233fc18","projectPage":"https://xiac20.github.io/SimRecon/","githubRepo":"https://github.com/xiac20/SimRecon","githubRepoAddedBy":"user","ai_summary":"SimRecon enables compositional scene reconstruction through a perception-generation-simulation pipeline with active viewpoint optimization and scene graph synthesizer modules to improve visual fidelity and physical plausibility.","ai_keywords":["compositional scene reconstruction","perception-generation-simulation pipeline","active viewpoint optimization","scene graph synthesizer","3D space","3D simulator","ScanNet dataset"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":146},"publishedAt":"2026-03-02T12:51:45.000Z","title":"SimRecon: SimReady Compositional Scene Reconstruction from Real Videos","summary":"Compositional scene reconstruction seeks to create object-centric representations rather than holistic scenes from real-world videos, which is natively applicable for simulation and interaction. Conventional compositional reconstruction approaches primarily emphasize on visual appearance and show limited generalization ability to real-world scenarios. In this paper, we propose SimRecon, a framework that realizes a \"Perception-Generation-Simulation\" pipeline towards cluttered scene reconstruction, which first conducts scene-level semantic reconstruction from video input, then performs single-object generation, and finally assembles these assets in the simulator. However, naively combining these three stages leads to visual infidelity of generated assets and physical implausibility of the final scene, a problem particularly severe for complex scenes. Thus, we further propose two bridging modules between the three stages to address this problem. To be specific, for the transition from Perception to Generation, critical for visual fidelity, we introduce Active Viewpoint Optimization, which actively searches in 3D space to acquire optimal projected images as conditions for single-object completion. Moreover, for the transition from Generation to Simulation, essential for physical plausibility, we propose a Scene Graph Synthesizer, which guides the construction from scratch in 3D simulators, mirroring the native, constructive principle of the real world. Extensive experiments on the ScanNet dataset validate our method's superior performance over previous state-of-the-art approaches.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2603.02133.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"SimRecon: SimReady Compositional Scene Reconstruction from Real Videos"}],"highlightedSummary":[{"type":"text","text":"Compositional scene reconstruction seeks to create object-centric representations rather than holistic scenes from real-world videos, which is natively applicable for simulation and interaction. Conventional compositional reconstruction approaches primarily emphasize on visual appearance and show limited generalization ability to real-world scenarios. In this paper, we propose SimRecon, a framework that realizes a \"Perception-Generation-Simulation\" pipeline towards cluttered scene reconstruction, which first conducts scene-level semantic reconstruction from video input, then performs single-object generation, and finally assembles these assets in the simulator. However, naively combining these three stages leads to visual infidelity of generated assets and physical implausibility of the final scene, a problem particularly severe for complex scenes. Thus, we further propose two bridging modules between the three stages to address this problem. To be specific, for the transition from Perception to Generation, critical for visual fidelity, we introduce Active Viewpoint Optimization, which actively searches in 3D space to acquire optimal projected images as conditions for single-object completion. "},{"type":"highlight","text":"Moreover, for the transition from Generation to Simulation, essential for physical plausibility, we propose a Scene Graph Synthesizer, which guides the construction from scratch in 3D simulators, mirroring the native, constructive principle of the real world."},{"type":"text","text":" Extensive experiments on the ScanNet dataset validate our method's superior performance over previous state-of-the-art approaches."}]},{"paper":{"id":"2512.17459","authors":[{"_id":"6949054634f46eaf46cbb3f6","user":{"_id":"69f9ba5925d387c2719592af","avatarUrl":"/avatars/89307591f8a36aa28e9426f7f609158a.svg","isPro":false,"fullname":"Tobias Sautter","user":"sautterto","type":"user","name":"sautterto"},"name":"Tobias Sautter","status":"claimed_verified","statusLastChangedAt":"2026-05-05T17:29:47.534Z","hidden":false},{"_id":"6949054634f46eaf46cbb3f7","user":{"_id":"631b6dbcbf1351ed2bd05be1","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/631b6dbcbf1351ed2bd05be1/_kzcDD4qCifsb-651_LmY.png","isPro":false,"fullname":"Jan-Niklas Dihlmann","user":"JDihlmann","type":"user","name":"JDihlmann"},"name":"Jan-Niklas Dihlmann","status":"claimed_verified","statusLastChangedAt":"2025-12-22T10:58:21.425Z","hidden":false},{"_id":"6949054634f46eaf46cbb3f8","name":"Hendrik P. A. Lensch","hidden":false}],"publishedAt":"2025-12-19T11:20:52.000Z","title":"3D-RE-GEN: 3D Reconstruction of Indoor Scenes with a Generative Framework","summary":"Recent advances in 3D scene generation produce visually appealing output, but current representations hinder artists' workflows that require modifiable 3D textured mesh scenes for visual effects and game development. Despite significant advances, current textured mesh scene reconstruction methods are far from artist ready, suffering from incorrect object decomposition, inaccurate spatial relationships, and missing backgrounds. We present 3D-RE-GEN, a compositional framework that reconstructs a single image into textured 3D objects and a background. We show that combining state of the art models from specific domains achieves state of the art scene reconstruction performance, addressing artists' requirements.\n  Our reconstruction pipeline integrates models for asset detection, reconstruction, and placement, pushing certain models beyond their originally intended domains. Obtaining occluded objects is treated as an image editing task with generative models to infer and reconstruct with scene level reasoning under consistent lighting and geometry. Unlike current methods, 3D-RE-GEN generates a comprehensive background that spatially constrains objects during optimization and provides a foundation for realistic lighting and simulation tasks in visual effects and games. To obtain physically realistic layouts, we employ a novel 4-DoF differentiable optimization that aligns reconstructed objects with the estimated ground plane. 3D-RE-GEN~achieves state of the art performance in single image 3D scene reconstruction, producing coherent, modifiable scenes through compositional generation guided by precise camera recovery and spatial optimization.","upvotes":12,"discussionId":"6949054634f46eaf46cbb3f9","projectPage":"https://3dregen.jdihlmann.com/","githubRepo":"https://github.com/cgtuebingen/3D-RE-GEN","githubRepoAddedBy":"user","ai_summary":"3D-RE-GEN presents a compositional framework for reconstructing single images into textured 3D scenes with improved object decomposition, spatial relationships, and background generation through integrated models and differentiable optimization.","ai_keywords":["3D scene generation","textured mesh scenes","image editing task","generative models","scene level reasoning","camera recovery","spatial optimization","4-DoF differentiable optimization","ground plane alignment"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":635},"publishedAt":"2025-12-19T06:20:52.000Z","title":"3D-RE-GEN: 3D Reconstruction of Indoor Scenes with a Generative Framework","summary":"Recent advances in 3D scene generation produce visually appealing output, but current representations hinder artists' workflows that require modifiable 3D textured mesh scenes for visual effects and game development. Despite significant advances, current textured mesh scene reconstruction methods are far from artist ready, suffering from incorrect object decomposition, inaccurate spatial relationships, and missing backgrounds. We present 3D-RE-GEN, a compositional framework that reconstructs a single image into textured 3D objects and a background. We show that combining state of the art models from specific domains achieves state of the art scene reconstruction performance, addressing artists' requirements.\n  Our reconstruction pipeline integrates models for asset detection, reconstruction, and placement, pushing certain models beyond their originally intended domains. Obtaining occluded objects is treated as an image editing task with generative models to infer and reconstruct with scene level reasoning under consistent lighting and geometry. Unlike current methods, 3D-RE-GEN generates a comprehensive background that spatially constrains objects during optimization and provides a foundation for realistic lighting and simulation tasks in visual effects and games. To obtain physically realistic layouts, we employ a novel 4-DoF differentiable optimization that aligns reconstructed objects with the estimated ground plane. 3D-RE-GEN~achieves state of the art performance in single image 3D scene reconstruction, producing coherent, modifiable scenes through compositional generation guided by precise camera recovery and spatial optimization.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2512.17459.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"3D-RE-GEN: 3D Reconstruction of Indoor Scenes with a Generative Framework"}],"highlightedSummary":[{"type":"text","text":"Recent advances in 3D scene generation produce visually appealing output, but current representations hinder artists' workflows that require modifiable 3D textured mesh scenes for visual effects and game development. Despite significant advances, current textured mesh scene reconstruction methods are far from artist ready, suffering from incorrect object decomposition, inaccurate spatial relationships, and missing backgrounds. "},{"type":"highlight","text":"We present 3D-RE-GEN, a compositional framework that reconstructs a single image into textured 3D objects and a background."},{"type":"text","text":" We show that combining state of the art models from specific domains achieves state of the art scene reconstruction performance, addressing artists' requirements.\n  Our reconstruction pipeline integrates models for asset detection, reconstruction, and placement, pushing certain models beyond their originally intended domains. Obtaining occluded objects is treated as an image editing task with generative models to infer and reconstruct with scene level reasoning under consistent lighting and geometry. Unlike current methods, 3D-RE-GEN generates a comprehensive background that spatially constrains objects during optimization and provides a foundation for realistic lighting and simulation tasks in visual effects and games. To obtain physically realistic layouts, we employ a novel 4-DoF differentiable optimization that aligns reconstructed objects with the estimated ground plane. 3D-RE-GEN~achieves state of the art performance in single image 3D scene reconstruction, producing coherent, modifiable scenes through compositional generation guided by precise camera recovery and spatial optimization."}]},{"paper":{"id":"2510.23880","authors":[{"_id":"69030d7d72739622ee92a901","name":"Hanke Chen","hidden":false},{"_id":"69030d7d72739622ee92a902","name":"Yuan Liu","hidden":false},{"_id":"69030d7d72739622ee92a903","name":"Minchen Li","hidden":false}],"publishedAt":"2025-10-27T21:40:31.000Z","title":"TRELLISWorld: Training-Free World Generation from Object Generators","summary":"Text-driven 3D scene generation holds promise for a wide range of\napplications, from virtual prototyping to AR/VR and simulation. However,\nexisting methods are often constrained to single-object generation, require\ndomain-specific training, or lack support for full 360-degree viewability. In\nthis work, we present a training-free approach to 3D scene synthesis by\nrepurposing general-purpose text-to-3D object diffusion models as modular tile\ngenerators. We reformulate scene generation as a multi-tile denoising problem,\nwhere overlapping 3D regions are independently generated and seamlessly blended\nvia weighted averaging. This enables scalable synthesis of large, coherent\nscenes while preserving local semantic control. Our method eliminates the need\nfor scene-level datasets or retraining, relies on minimal heuristics, and\ninherits the generalization capabilities of object-level priors. We demonstrate\nthat our approach supports diverse scene layouts, efficient generation, and\nflexible editing, establishing a simple yet powerful foundation for\ngeneral-purpose, language-driven 3D scene construction.","upvotes":2,"discussionId":"69030d7e72739622ee92a904","ai_summary":"A text-driven approach to 3D scene generation uses modular tile generators derived from text-to-3D object diffusion models to create large, coherent scenes with minimal heuristics and without scene-level datasets.","ai_keywords":["text-to-3D object diffusion models","multi-tile denoising","weighted averaging","scene generation","general-purpose","language-driven 3D scene construction"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-10-27T17:40:31.000Z","title":"TRELLISWorld: Training-Free World Generation from Object Generators","summary":"Text-driven 3D scene generation holds promise for a wide range of\napplications, from virtual prototyping to AR/VR and simulation. However,\nexisting methods are often constrained to single-object generation, require\ndomain-specific training, or lack support for full 360-degree viewability. In\nthis work, we present a training-free approach to 3D scene synthesis by\nrepurposing general-purpose text-to-3D object diffusion models as modular tile\ngenerators. We reformulate scene generation as a multi-tile denoising problem,\nwhere overlapping 3D regions are independently generated and seamlessly blended\nvia weighted averaging. This enables scalable synthesis of large, coherent\nscenes while preserving local semantic control. Our method eliminates the need\nfor scene-level datasets or retraining, relies on minimal heuristics, and\ninherits the generalization capabilities of object-level priors. We demonstrate\nthat our approach supports diverse scene layouts, efficient generation, and\nflexible editing, establishing a simple yet powerful foundation for\ngeneral-purpose, language-driven 3D scene construction.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2510.23880.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"TRELLISWorld: Training-Free World Generation from Object Generators"}],"highlightedSummary":[{"type":"text","text":"Text-driven 3D scene generation holds promise for a wide range of\napplications, from virtual prototyping to AR/VR and simulation. However,\nexisting methods are often constrained to single-object generation, require\ndomain-specific training, or lack support for full 360-degree viewability. In\nthis work, we present a training-free approach to 3D scene synthesis by\nrepurposing general-purpose text-to-3D object diffusion models as modular tile\ngenerators. We reformulate scene generation as a multi-tile denoising problem,\nwhere overlapping 3D regions are independently generated and seamlessly blended\nvia weighted averaging. This enables scalable synthesis of large, coherent\nscenes while preserving local semantic control. Our method eliminates the need\nfor scene-level datasets or retraining, relies on minimal heuristics, and\ninherits the generalization capabilities of object-level priors. "},{"type":"highlight","text":"We demonstrate\nthat our approach supports diverse scene layouts, efficient generation, and\nflexible editing, establishing a simple yet powerful foundation for\ngeneral-purpose, language-driven 3D scene construction."}]},{"paper":{"id":"2603.05908","authors":[{"_id":"69b0fa291f7902d47f6dfa8a","name":"Zidian Qiu","hidden":false},{"_id":"69b0fa291f7902d47f6dfa8b","name":"Ancong Wu","hidden":false}],"publishedAt":"2026-03-06T04:54:23.000Z","title":"Pano3DComposer: Feed-Forward Compositional 3D Scene Generation from Single Panoramic Image","summary":"Current compositional image-to-3D scene generation approaches construct 3D scenes by time-consuming iterative layout optimization or inflexible joint object-layout generation. Moreover, most methods rely on limited field-of-view perspective images, hindering the creation of complete 360-degree environments. To address these limitations, we design Pano3DComposer, an efficient feed-forward framework for panoramic images. To decouple object generation from layout estimation, we propose a plug-and-play Object-World Transformation Predictor. This module converts the 3D objects generated by off-the-shelf image-to-3D models from local to world coordinates. To achieve this, we adapt the VGGT architecture to Alignment-VGGT by using target object crop, multi-view object renderings and camera parameters to predict the transformation. The predictor is trained using pseudo-geometric supervision to address the shape discrepancy between generated and ground-truth objects. For input images from unseen domains, we further introduce a Coarse-to-Fine (C2F) alignment mechanism for Pano3DComposer that iteratively refines geometric consistency with feedback of scene rendering. Our method achieves superior geometric accuracy for image/text-to-3D tasks on synthetic and real-world datasets. It can generate a high-fidelity 3D scene in approximately 20 seconds on an RTX 4090 GPU. Project page: https://qiuzidian.github.io/pano3dcomposer-page/.","upvotes":0,"discussionId":"69b0fa2a1f7902d47f6dfa8c","ai_summary":"Pano3DComposer enables efficient panoramic 3D scene generation through object-world transformation prediction and coarse-to-fine alignment mechanisms.","ai_keywords":["Object-World Transformation Predictor","VGGT architecture","Alignment-VGGT","pseudo-geometric supervision","Coarse-to-Fine alignment mechanism","panoramic images","3D scene generation","image-to-3D models","geometric accuracy","scene rendering"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2026-03-05T23:54:23.000Z","title":"Pano3DComposer: Feed-Forward Compositional 3D Scene Generation from Single Panoramic Image","summary":"Current compositional image-to-3D scene generation approaches construct 3D scenes by time-consuming iterative layout optimization or inflexible joint object-layout generation. Moreover, most methods rely on limited field-of-view perspective images, hindering the creation of complete 360-degree environments. To address these limitations, we design Pano3DComposer, an efficient feed-forward framework for panoramic images. To decouple object generation from layout estimation, we propose a plug-and-play Object-World Transformation Predictor. This module converts the 3D objects generated by off-the-shelf image-to-3D models from local to world coordinates. To achieve this, we adapt the VGGT architecture to Alignment-VGGT by using target object crop, multi-view object renderings and camera parameters to predict the transformation. The predictor is trained using pseudo-geometric supervision to address the shape discrepancy between generated and ground-truth objects. For input images from unseen domains, we further introduce a Coarse-to-Fine (C2F) alignment mechanism for Pano3DComposer that iteratively refines geometric consistency with feedback of scene rendering. Our method achieves superior geometric accuracy for image/text-to-3D tasks on synthetic and real-world datasets. It can generate a high-fidelity 3D scene in approximately 20 seconds on an RTX 4090 GPU. Project page: https://qiuzidian.github.io/pano3dcomposer-page/.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2603.05908.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Pano3DComposer: Feed-Forward Compositional 3D Scene Generation from Single Panoramic Image"}],"highlightedSummary":[{"type":"highlight","text":"Current compositional image-to-3D scene generation approaches construct 3D scenes by time-consuming iterative layout optimization or inflexible joint object-layout generation."},{"type":"text","text":" Moreover, most methods rely on limited field-of-view perspective images, hindering the creation of complete 360-degree environments. To address these limitations, we design Pano3DComposer, an efficient feed-forward framework for panoramic images. To decouple object generation from layout estimation, we propose a plug-and-play Object-World Transformation Predictor. This module converts the 3D objects generated by off-the-shelf image-to-3D models from local to world coordinates. To achieve this, we adapt the VGGT architecture to Alignment-VGGT by using target object crop, multi-view object renderings and camera parameters to predict the transformation. The predictor is trained using pseudo-geometric supervision to address the shape discrepancy between generated and ground-truth objects. For input images from unseen domains, we further introduce a Coarse-to-Fine (C2F) alignment mechanism for Pano3DComposer that iteratively refines geometric consistency with feedback of scene rendering. Our method achieves superior geometric accuracy for image/text-to-3D tasks on synthetic and real-world datasets. It can generate a high-fidelity 3D scene in approximately 20 seconds on an RTX 4090 GPU. Project page: https://qiuzidian.github.io/pano3dcomposer-page/."}]},{"paper":{"id":"2504.06178","authors":[{"_id":"67f8d79af5889583ce19b5a6","user":{"_id":"67da3534e6387c9c9bc954b3","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/no-auth/1IiNF_DgWQcvMYIlW8ikC.png","isPro":false,"fullname":"HU YUJIA","user":"Cicici1109","type":"user","name":"Cicici1109"},"name":"Yujia Hu","status":"claimed_verified","statusLastChangedAt":"2025-04-11T12:01:09.243Z","hidden":false},{"_id":"67f8d79af5889583ce19b5a7","user":{"_id":"64b929308b53fb5dbd059ce3","avatarUrl":"/avatars/564c44cf45db794747a96f79f30ecd91.svg","isPro":false,"fullname":"Liu Songhua","user":"Huage001","type":"user","name":"Huage001"},"name":"Songhua Liu","status":"claimed_verified","statusLastChangedAt":"2026-03-17T09:38:24.284Z","hidden":false},{"_id":"67f8d79af5889583ce19b5a8","user":{"_id":"634cfebc350bcee9bed20a4d","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/634cfebc350bcee9bed20a4d/fN47nN5rhw-HJaFLBZWQy.png","isPro":false,"fullname":"Xingyi Yang","user":"adamdad","type":"user","name":"adamdad"},"name":"Xingyi Yang","status":"claimed_verified","statusLastChangedAt":"2025-06-05T07:29:34.404Z","hidden":false},{"_id":"67f8d79af5889583ce19b5a9","name":"Xinchao Wang","hidden":false}],"publishedAt":"2025-04-08T16:20:51.000Z","title":"Flash Sculptor: Modular 3D Worlds from Objects","summary":"Existing text-to-3D and image-to-3D models often struggle with complex scenes\ninvolving multiple objects and intricate interactions. Although some recent\nattempts have explored such compositional scenarios, they still require an\nextensive process of optimizing the entire layout, which is highly cumbersome\nif not infeasible at all. To overcome these challenges, we propose Flash\nSculptor in this paper, a simple yet effective framework for compositional 3D\nscene/object reconstruction from a single image. At the heart of Flash Sculptor\nlies a divide-and-conquer strategy, which decouples compositional scene\nreconstruction into a sequence of sub-tasks, including handling the appearance,\nrotation, scale, and translation of each individual instance. Specifically, for\nrotation, we introduce a coarse-to-fine scheme that brings the best of both\nworlds--efficiency and accuracy--while for translation, we develop an\noutlier-removal-based algorithm that ensures robust and precise parameters in a\nsingle step, without any iterative optimization. Extensive experiments\ndemonstrate that Flash Sculptor achieves at least a 3 times speedup over\nexisting compositional 3D methods, while setting new benchmarks in\ncompositional 3D reconstruction performance. Codes are available at\nhttps://github.com/YujiaHu1109/Flash-Sculptor.","upvotes":0,"discussionId":"67f8d79cf5889583ce19b617","githubRepo":"https://github.com/YujiaHu1109/Flash-Sculptor","githubRepoAddedBy":"user","ai_summary":"Flash Sculptor efficiently reconstructs compositional 3D scenes from a single image through a divide-and-conquer strategy.","ai_keywords":["divide-and-conquer","coarse-to-fine","outlier-removal-based algorithm","compositional 3D scene reconstruction"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":35},"publishedAt":"2025-04-08T12:20:51.000Z","title":"Flash Sculptor: Modular 3D Worlds from Objects","summary":"Existing text-to-3D and image-to-3D models often struggle with complex scenes\ninvolving multiple objects and intricate interactions. Although some recent\nattempts have explored such compositional scenarios, they still require an\nextensive process of optimizing the entire layout, which is highly cumbersome\nif not infeasible at all. To overcome these challenges, we propose Flash\nSculptor in this paper, a simple yet effective framework for compositional 3D\nscene/object reconstruction from a single image. At the heart of Flash Sculptor\nlies a divide-and-conquer strategy, which decouples compositional scene\nreconstruction into a sequence of sub-tasks, including handling the appearance,\nrotation, scale, and translation of each individual instance. Specifically, for\nrotation, we introduce a coarse-to-fine scheme that brings the best of both\nworlds--efficiency and accuracy--while for translation, we develop an\noutlier-removal-based algorithm that ensures robust and precise parameters in a\nsingle step, without any iterative optimization. Extensive experiments\ndemonstrate that Flash Sculptor achieves at least a 3 times speedup over\nexisting compositional 3D methods, while setting new benchmarks in\ncompositional 3D reconstruction performance. Codes are available at\nhttps://github.com/YujiaHu1109/Flash-Sculptor.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2504.06178.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Flash Sculptor: Modular 3D Worlds from Objects"}],"highlightedSummary":[{"type":"text","text":"Existing text-to-3D and image-to-3D models often struggle with complex scenes\ninvolving multiple objects and intricate interactions. Although some recent\nattempts have explored such compositional scenarios, they still require an\nextensive process of optimizing the entire layout, which is highly cumbersome\nif not infeasible at all. "},{"type":"highlight","text":"To overcome these challenges, we propose Flash\nSculptor in this paper, a simple yet effective framework for compositional 3D\nscene/object reconstruction from a single image."},{"type":"text","text":" At the heart of Flash Sculptor\nlies a divide-and-conquer strategy, which decouples compositional scene\nreconstruction into a sequence of sub-tasks, including handling the appearance,\nrotation, scale, and translation of each individual instance. Specifically, for\nrotation, we introduce a coarse-to-fine scheme that brings the best of both\nworlds--efficiency and accuracy--while for translation, we develop an\noutlier-removal-based algorithm that ensures robust and precise parameters in a\nsingle step, without any iterative optimization. Extensive experiments\ndemonstrate that Flash Sculptor achieves at least a 3 times speedup over\nexisting compositional 3D methods, while setting new benchmarks in\ncompositional 3D reconstruction performance. Codes are available at\nhttps://github.com/YujiaHu1109/Flash-Sculptor."}]},{"paper":{"id":"2508.15769","authors":[{"_id":"68a7cd5939413c456c05b09e","user":{"_id":"666a5ca53315aa1fa8061ef1","avatarUrl":"/avatars/da5ce98959f553c7d25675acae55b6fe.svg","isPro":false,"fullname":"Mengmouxu","user":"Mengmouxu","type":"user","name":"Mengmouxu"},"name":"Yanxu Meng","status":"claimed_verified","statusLastChangedAt":"2025-08-26T19:06:35.644Z","hidden":false},{"_id":"68a7cd5939413c456c05b09f","user":{"_id":"632c7a0d1d303f5f9acf01b8","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/632c7a0d1d303f5f9acf01b8/T010IFuCp6UaOeIyWhbCk.jpeg","isPro":false,"fullname":"Haoning Wu","user":"haoningwu","type":"user","name":"haoningwu"},"name":"Haoning Wu","status":"claimed_verified","statusLastChangedAt":"2025-08-22T07:22:01.228Z","hidden":false},{"_id":"68a7cd5939413c456c05b0a0","name":"Ya Zhang","hidden":false},{"_id":"68a7cd5939413c456c05b0a1","user":{"_id":"63f1841f2f7c0152e8748ec4","avatarUrl":"/avatars/5398658c7d0cf556531a625a4ca5d18a.svg","isPro":false,"fullname":"Xie","user":"Weidi","type":"user","name":"Weidi"},"name":"Weidi Xie","status":"admin_assigned","statusLastChangedAt":"2025-08-22T08:27:57.412Z","hidden":false}],"publishedAt":"2025-08-21T17:59:16.000Z","title":"SceneGen: Single-Image 3D Scene Generation in One Feedforward Pass","summary":"3D content generation has recently attracted significant research interest\ndue to its applications in VR/AR and embodied AI. In this work, we address the\nchallenging task of synthesizing multiple 3D assets within a single scene\nimage. Concretely, our contributions are fourfold: (i) we present SceneGen, a\nnovel framework that takes a scene image and corresponding object masks as\ninput, simultaneously producing multiple 3D assets with geometry and texture.\nNotably, SceneGen operates with no need for optimization or asset retrieval;\n(ii) we introduce a novel feature aggregation module that integrates local and\nglobal scene information from visual and geometric encoders within the feature\nextraction module. Coupled with a position head, this enables the generation of\n3D assets and their relative spatial positions in a single feedforward pass;\n(iii) we demonstrate SceneGen's direct extensibility to multi-image input\nscenarios. Despite being trained solely on single-image inputs, our\narchitectural design enables improved generation performance with multi-image\ninputs; and (iv) extensive quantitative and qualitative evaluations confirm the\nefficiency and robust generation abilities of our approach. We believe this\nparadigm offers a novel solution for high-quality 3D content generation,\npotentially advancing its practical applications in downstream tasks. The code\nand model will be publicly available at: https://mengmouxu.github.io/SceneGen.","upvotes":19,"discussionId":"68a7cd5939413c456c05b0a2","projectPage":"https://mengmouxu.github.io/SceneGen/","githubRepo":"https://github.com/Mengmouxu/SceneGen","githubRepoAddedBy":"user","ai_summary":"SceneGen generates multiple 3D assets from a single scene image using a novel framework that integrates local and global scene information, enabling efficient and robust 3D content creation.","ai_keywords":["SceneGen","feature aggregation module","visual encoders","geometric encoders","position head","multi-image input","3D content generation","VR/AR","embodied AI"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":516},"publishedAt":"2025-08-21T13:59:16.000Z","title":"SceneGen: Single-Image 3D Scene Generation in One Feedforward Pass","summary":"3D content generation has recently attracted significant research interest\ndue to its applications in VR/AR and embodied AI. In this work, we address the\nchallenging task of synthesizing multiple 3D assets within a single scene\nimage. Concretely, our contributions are fourfold: (i) we present SceneGen, a\nnovel framework that takes a scene image and corresponding object masks as\ninput, simultaneously producing multiple 3D assets with geometry and texture.\nNotably, SceneGen operates with no need for optimization or asset retrieval;\n(ii) we introduce a novel feature aggregation module that integrates local and\nglobal scene information from visual and geometric encoders within the feature\nextraction module. Coupled with a position head, this enables the generation of\n3D assets and their relative spatial positions in a single feedforward pass;\n(iii) we demonstrate SceneGen's direct extensibility to multi-image input\nscenarios. Despite being trained solely on single-image inputs, our\narchitectural design enables improved generation performance with multi-image\ninputs; and (iv) extensive quantitative and qualitative evaluations confirm the\nefficiency and robust generation abilities of our approach. We believe this\nparadigm offers a novel solution for high-quality 3D content generation,\npotentially advancing its practical applications in downstream tasks. The code\nand model will be publicly available at: https://mengmouxu.github.io/SceneGen.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2508.15769.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"SceneGen: Single-Image 3D Scene Generation in One Feedforward Pass"}],"highlightedSummary":[{"type":"text","text":"3D content generation has recently attracted significant research interest\ndue to its applications in VR/AR and embodied AI. In this work, we address the\nchallenging task of synthesizing multiple 3D assets within a single scene\nimage. Concretely, our contributions are fourfold: (i) we present SceneGen, a\nnovel framework that takes a scene image and corresponding object masks as\ninput, simultaneously producing multiple 3D assets with geometry and texture.\nNotably, SceneGen operates with no need for optimization or asset retrieval;\n(ii) we introduce a novel feature aggregation module that integrates local and\nglobal scene information from visual and geometric encoders within the feature\nextraction module. Coupled with a position head, this enables the generation of\n3D assets and their relative spatial positions in a single feedforward pass;\n(iii) we demonstrate SceneGen's direct extensibility to multi-image input\nscenarios. "},{"type":"highlight","text":"Despite being trained solely on single-image inputs, our\narchitectural design enables improved generation performance with multi-image\ninputs; and (iv) extensive quantitative and qualitative evaluations confirm the\nefficiency and robust generation abilities of our approach."},{"type":"text","text":" We believe this\nparadigm offers a novel solution for high-quality 3D content generation,\npotentially advancing its practical applications in downstream tasks. The code\nand model will be publicly available at: https://mengmouxu.github.io/SceneGen."}]},{"paper":{"id":"2412.08645","authors":[{"_id":"675b1648634bd80c4c1d84f8","user":{"_id":"675fccea42bc7570939a3642","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/no-auth/x081h2YwSs9kIUgA5me8z.png","isPro":false,"fullname":"Daniel Winter","user":"danielwinter","type":"user","name":"danielwinter"},"name":"Daniel Winter","status":"admin_assigned","statusLastChangedAt":"2024-12-16T10:16:31.261Z","hidden":false},{"_id":"675b1648634bd80c4c1d84f9","name":"Asaf Shul","hidden":false},{"_id":"675b1648634bd80c4c1d84fa","name":"Matan Cohen","hidden":false},{"_id":"675b1648634bd80c4c1d84fb","name":"Dana Berman","hidden":false},{"_id":"675b1648634bd80c4c1d84fc","name":"Yael Pritch","hidden":false},{"_id":"675b1648634bd80c4c1d84fd","name":"Alex Rav-Acha","hidden":false},{"_id":"675b1648634bd80c4c1d84fe","user":{"_id":"646cfc3b4220471ca0c56b20","avatarUrl":"/avatars/19d6ab141ec2cd25c1c3b45fd8f69910.svg","isPro":false,"fullname":"Yedid Hoshen","user":"yedid","type":"user","name":"yedid"},"name":"Yedid Hoshen","status":"claimed_verified","statusLastChangedAt":"2024-12-16T09:42:31.672Z","hidden":false}],"publishedAt":"2024-12-11T18:59:53.000Z","title":"ObjectMate: A Recurrence Prior for Object Insertion and Subject-Driven\n  Generation","summary":"This paper introduces a tuning-free method for both object insertion and\nsubject-driven generation. The task involves composing an object, given\nmultiple views, into a scene specified by either an image or text. Existing\nmethods struggle to fully meet the task's challenging objectives: (i)\nseamlessly composing the object into the scene with photorealistic pose and\nlighting, and (ii) preserving the object's identity. We hypothesize that\nachieving these goals requires large scale supervision, but manually collecting\nsufficient data is simply too expensive. The key observation in this paper is\nthat many mass-produced objects recur across multiple images of large unlabeled\ndatasets, in different scenes, poses, and lighting conditions. We use this\nobservation to create massive supervision by retrieving sets of diverse views\nof the same object. This powerful paired dataset enables us to train a\nstraightforward text-to-image diffusion architecture to map the object and\nscene descriptions to the composited image. We compare our method, ObjectMate,\nwith state-of-the-art methods for object insertion and subject-driven\ngeneration, using a single or multiple references. Empirically, ObjectMate\nachieves superior identity preservation and more photorealistic composition.\nDifferently from many other multi-reference methods, ObjectMate does not\nrequire slow test-time tuning.","upvotes":12,"discussionId":"675b164c634bd80c4c1d85e6","projectPage":"https://object-mate.com","ai_summary":"A straightforward text-to-image diffusion model trained on a massive dataset of diverse object views achieves superior identity preservation and photorealistic composition without requiring test-time tuning.","ai_keywords":["diffusion architecture","text-to-image","object insertion","subject-driven generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-12-11T13:59:53.000Z","title":"ObjectMate: A Recurrence Prior for Object Insertion and Subject-Driven\n  Generation","summary":"This paper introduces a tuning-free method for both object insertion and\nsubject-driven generation. The task involves composing an object, given\nmultiple views, into a scene specified by either an image or text. Existing\nmethods struggle to fully meet the task's challenging objectives: (i)\nseamlessly composing the object into the scene with photorealistic pose and\nlighting, and (ii) preserving the object's identity. We hypothesize that\nachieving these goals requires large scale supervision, but manually collecting\nsufficient data is simply too expensive. The key observation in this paper is\nthat many mass-produced objects recur across multiple images of large unlabeled\ndatasets, in different scenes, poses, and lighting conditions. We use this\nobservation to create massive supervision by retrieving sets of diverse views\nof the same object. This powerful paired dataset enables us to train a\nstraightforward text-to-image diffusion architecture to map the object and\nscene descriptions to the composited image. We compare our method, ObjectMate,\nwith state-of-the-art methods for object insertion and subject-driven\ngeneration, using a single or multiple references. Empirically, ObjectMate\nachieves superior identity preservation and more photorealistic composition.\nDifferently from many other multi-reference methods, ObjectMate does not\nrequire slow test-time tuning.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2412.08645.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"ObjectMate: A Recurrence Prior for Object Insertion and Subject-Driven\n  Generation"}],"highlightedSummary":[{"type":"text","text":"This paper introduces a tuning-free method for both object insertion and\nsubject-driven generation. "},{"type":"highlight","text":"The task involves composing an object, given\nmultiple views, into a scene specified by either an image or text."},{"type":"text","text":" Existing\nmethods struggle to fully meet the task's challenging objectives: (i)\nseamlessly composing the object into the scene with photorealistic pose and\nlighting, and (ii) preserving the object's identity. We hypothesize that\nachieving these goals requires large scale supervision, but manually collecting\nsufficient data is simply too expensive. The key observation in this paper is\nthat many mass-produced objects recur across multiple images of large unlabeled\ndatasets, in different scenes, poses, and lighting conditions. We use this\nobservation to create massive supervision by retrieving sets of diverse views\nof the same object. This powerful paired dataset enables us to train a\nstraightforward text-to-image diffusion architecture to map the object and\nscene descriptions to the composited image. We compare our method, ObjectMate,\nwith state-of-the-art methods for object insertion and subject-driven\ngeneration, using a single or multiple references. Empirically, ObjectMate\nachieves superior identity preservation and more photorealistic composition.\nDifferently from many other multi-reference methods, ObjectMate does not\nrequire slow test-time tuning."}]},{"paper":{"id":"2505.05474","authors":[{"_id":"681d7b5ae27a030c96a28bde","user":{"_id":"672392c4a4c4381cefc06416","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/672392c4a4c4381cefc06416/f5eNT7WNPYWRRchm49Lco.jpeg","isPro":false,"fullname":"Beichen Wen","user":"wenbc21","type":"user","name":"wenbc21"},"name":"Beichen Wen","status":"admin_assigned","statusLastChangedAt":"2025-05-09T08:56:55.900Z","hidden":false},{"_id":"681d7b5ae27a030c96a28bdf","user":{"_id":"63f47b5321eb234ab739e91a","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/63f47b5321eb234ab739e91a/vWfFNVtMkHl8gieha5PPd.jpeg","isPro":false,"fullname":"Haozhe Xie","user":"hzxie","type":"user","name":"hzxie"},"name":"Haozhe Xie","status":"admin_assigned","statusLastChangedAt":"2025-05-09T08:57:02.341Z","hidden":false},{"_id":"681d7b5ae27a030c96a28be0","user":{"_id":"62fc8cf7ee999004b5a8b982","avatarUrl":"/avatars/6c5dda9e58747054a989f077a078f3dc.svg","isPro":false,"fullname":"Zhaoxi Chen","user":"FrozenBurning","type":"user","name":"FrozenBurning"},"name":"Zhaoxi Chen","status":"admin_assigned","statusLastChangedAt":"2025-05-09T08:57:08.543Z","hidden":false},{"_id":"681d7b5ae27a030c96a28be1","name":"Fangzhou Hong","hidden":false},{"_id":"681d7b5ae27a030c96a28be2","user":{"_id":"62ab1ac1d48b4d8b048a3473","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1656826685333-62ab1ac1d48b4d8b048a3473.png","isPro":false,"fullname":"Ziwei Liu","user":"liuziwei7","type":"user","name":"liuziwei7"},"name":"Ziwei Liu","status":"admin_assigned","statusLastChangedAt":"2025-05-09T08:57:27.473Z","hidden":false}],"publishedAt":"2025-05-08T17:59:54.000Z","title":"3D Scene Generation: A Survey","summary":"3D scene generation seeks to synthesize spatially structured, semantically\nmeaningful, and photorealistic environments for applications such as immersive\nmedia, robotics, autonomous driving, and embodied AI. Early methods based on\nprocedural rules offered scalability but limited diversity. Recent advances in\ndeep generative models (e.g., GANs, diffusion models) and 3D representations\n(e.g., NeRF, 3D Gaussians) have enabled the learning of real-world scene\ndistributions, improving fidelity, diversity, and view consistency. Recent\nadvances like diffusion models bridge 3D scene synthesis and photorealism by\nreframing generation as image or video synthesis problems. This survey provides\na systematic overview of state-of-the-art approaches, organizing them into four\nparadigms: procedural generation, neural 3D-based generation, image-based\ngeneration, and video-based generation. We analyze their technical foundations,\ntrade-offs, and representative results, and review commonly used datasets,\nevaluation protocols, and downstream applications. We conclude by discussing\nkey challenges in generation capacity, 3D representation, data and annotations,\nand evaluation, and outline promising directions including higher fidelity,\nphysics-aware and interactive generation, and unified perception-generation\nmodels. This review organizes recent advances in 3D scene generation and\nhighlights promising directions at the intersection of generative AI, 3D\nvision, and embodied intelligence. To track ongoing developments, we maintain\nan up-to-date project page:\nhttps://github.com/hzxie/Awesome-3D-Scene-Generation.","upvotes":22,"discussionId":"681d7b5be27a030c96a28c29","githubRepo":"https://github.com/hzxie/Awesome-3D-Scene-Generation","githubRepoAddedBy":"user","ai_summary":"This survey examines the state-of-the-art methods in 3D scene generation, categorizing them into procedural, neural 3D, image-based, and video-based paradigms, and discusses current challenges and future directions.","ai_keywords":["deep generative models","GANs","diffusion models","NeRF","3D Gaussians","procedural generation","neural 3D-based generation","image-based generation","video-based generation","generative AI","3D vision","embodied intelligence"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":1099},"publishedAt":"2025-05-08T13:59:54.000Z","title":"3D Scene Generation: A Survey","summary":"3D scene generation seeks to synthesize spatially structured, semantically\nmeaningful, and photorealistic environments for applications such as immersive\nmedia, robotics, autonomous driving, and embodied AI. Early methods based on\nprocedural rules offered scalability but limited diversity. Recent advances in\ndeep generative models (e.g., GANs, diffusion models) and 3D representations\n(e.g., NeRF, 3D Gaussians) have enabled the learning of real-world scene\ndistributions, improving fidelity, diversity, and view consistency. Recent\nadvances like diffusion models bridge 3D scene synthesis and photorealism by\nreframing generation as image or video synthesis problems. This survey provides\na systematic overview of state-of-the-art approaches, organizing them into four\nparadigms: procedural generation, neural 3D-based generation, image-based\ngeneration, and video-based generation. We analyze their technical foundations,\ntrade-offs, and representative results, and review commonly used datasets,\nevaluation protocols, and downstream applications. We conclude by discussing\nkey challenges in generation capacity, 3D representation, data and annotations,\nand evaluation, and outline promising directions including higher fidelity,\nphysics-aware and interactive generation, and unified perception-generation\nmodels. This review organizes recent advances in 3D scene generation and\nhighlights promising directions at the intersection of generative AI, 3D\nvision, and embodied intelligence. To track ongoing developments, we maintain\nan up-to-date project page:\nhttps://github.com/hzxie/Awesome-3D-Scene-Generation.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2505.05474.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"3D Scene Generation: A Survey"}],"highlightedSummary":[{"type":"highlight","text":"3D scene generation seeks to synthesize spatially structured, semantically\nmeaningful, and photorealistic environments for applications such as immersive\nmedia, robotics, autonomous driving, and embodied AI."},{"type":"text","text":" Early methods based on\nprocedural rules offered scalability but limited diversity. Recent advances in\ndeep generative models (e.g., GANs, diffusion models) and 3D representations\n(e.g., NeRF, 3D Gaussians) have enabled the learning of real-world scene\ndistributions, improving fidelity, diversity, and view consistency. Recent\nadvances like diffusion models bridge 3D scene synthesis and photorealism by\nreframing generation as image or video synthesis problems. This survey provides\na systematic overview of state-of-the-art approaches, organizing them into four\nparadigms: procedural generation, neural 3D-based generation, image-based\ngeneration, and video-based generation. We analyze their technical foundations,\ntrade-offs, and representative results, and review commonly used datasets,\nevaluation protocols, and downstream applications. We conclude by discussing\nkey challenges in generation capacity, 3D representation, data and annotations,\nand evaluation, and outline promising directions including higher fidelity,\nphysics-aware and interactive generation, and unified perception-generation\nmodels. This review organizes recent advances in 3D scene generation and\nhighlights promising directions at the intersection of generative AI, 3D\nvision, and embodied intelligence. To track ongoing developments, we maintain\nan up-to-date project page:\nhttps://github.com/hzxie/Awesome-3D-Scene-Generation."}]},{"paper":{"id":"2410.12669","authors":[{"_id":"673dd9dfc19f3f69352f8cfc","name":"Dewei Zhou","hidden":false},{"_id":"673dd9dfc19f3f69352f8cfd","user":{"_id":"64e99fc07e2ec711a7138262","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64e99fc07e2ec711a7138262/9-Y-ZFaUhDpi_8X4oRO5E.jpeg","isPro":true,"fullname":"Ji Xie","user":"sanaka87","type":"user","name":"sanaka87"},"name":"Ji Xie","status":"claimed_verified","statusLastChangedAt":"2025-01-15T08:49:40.478Z","hidden":false},{"_id":"673dd9dfc19f3f69352f8cfe","name":"Zongxin Yang","hidden":false},{"_id":"673dd9dfc19f3f69352f8cff","name":"Yi Yang","hidden":false}],"publishedAt":"2024-10-16T15:34:13.000Z","title":"3DIS: Depth-Driven Decoupled Instance Synthesis for Text-to-Image\n  Generation","summary":"The increasing demand for controllable outputs in text-to-image generation\nhas spurred advancements in multi-instance generation (MIG), allowing users to\ndefine both instance layouts and attributes. However, unlike image-conditional\ngeneration methods such as ControlNet, MIG techniques have not been widely\nadopted in state-of-the-art models like SD2 and SDXL, primarily due to the\nchallenge of building robust renderers that simultaneously handle instance\npositioning and attribute rendering. In this paper, we introduce Depth-Driven\nDecoupled Instance Synthesis (3DIS), a novel framework that decouples the MIG\nprocess into two stages: (i) generating a coarse scene depth map for accurate\ninstance positioning and scene composition, and (ii) rendering fine-grained\nattributes using pre-trained ControlNet on any foundational model, without\nadditional training. Our 3DIS framework integrates a custom adapter into LDM3D\nfor precise depth-based layouts and employs a finetuning-free method for\nenhanced instance-level attribute rendering. Extensive experiments on\nCOCO-Position and COCO-MIG benchmarks demonstrate that 3DIS significantly\noutperforms existing methods in both layout precision and attribute rendering.\nNotably, 3DIS offers seamless compatibility with diverse foundational models,\nproviding a robust, adaptable solution for advanced multi-instance generation.\nThe code is available at: https://github.com/limuloo/3DIS.","upvotes":1,"discussionId":"673dd9e1c19f3f69352f8db8","githubRepo":"https://github.com/limuloo/3DIS","githubRepoAddedBy":"auto","ai_summary":"3DIS framework decouples multi-instance generation into depth mapping and attribute rendering, outperforming existing methods in layout precision and attribute rendering.","ai_keywords":["multi-instance generation","Depth-Driven Decoupled Instance Synthesis","LDM3D","coarse scene depth map","fine-grained attributes","ControlNet","COCO-Position","COCO-MIG"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":260},"publishedAt":"2024-10-16T11:34:13.000Z","title":"3DIS: Depth-Driven Decoupled Instance Synthesis for Text-to-Image\n  Generation","summary":"The increasing demand for controllable outputs in text-to-image generation\nhas spurred advancements in multi-instance generation (MIG), allowing users to\ndefine both instance layouts and attributes. However, unlike image-conditional\ngeneration methods such as ControlNet, MIG techniques have not been widely\nadopted in state-of-the-art models like SD2 and SDXL, primarily due to the\nchallenge of building robust renderers that simultaneously handle instance\npositioning and attribute rendering. In this paper, we introduce Depth-Driven\nDecoupled Instance Synthesis (3DIS), a novel framework that decouples the MIG\nprocess into two stages: (i) generating a coarse scene depth map for accurate\ninstance positioning and scene composition, and (ii) rendering fine-grained\nattributes using pre-trained ControlNet on any foundational model, without\nadditional training. Our 3DIS framework integrates a custom adapter into LDM3D\nfor precise depth-based layouts and employs a finetuning-free method for\nenhanced instance-level attribute rendering. Extensive experiments on\nCOCO-Position and COCO-MIG benchmarks demonstrate that 3DIS significantly\noutperforms existing methods in both layout precision and attribute rendering.\nNotably, 3DIS offers seamless compatibility with diverse foundational models,\nproviding a robust, adaptable solution for advanced multi-instance generation.\nThe code is available at: https://github.com/limuloo/3DIS.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2410.12669.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"3DIS: Depth-Driven Decoupled Instance Synthesis for Text-to-Image\n  Generation"}],"highlightedSummary":[{"type":"text","text":"The increasing demand for controllable outputs in text-to-image generation\nhas spurred advancements in multi-instance generation (MIG), allowing users to\ndefine both instance layouts and attributes. However, unlike image-conditional\ngeneration methods such as ControlNet, MIG techniques have not been widely\nadopted in state-of-the-art models like SD2 and SDXL, primarily due to the\nchallenge of building robust renderers that simultaneously handle instance\npositioning and attribute rendering. "},{"type":"highlight","text":"In this paper, we introduce Depth-Driven\nDecoupled Instance Synthesis (3DIS), a novel framework that decouples the MIG\nprocess into two stages: (i) generating a coarse scene depth map for accurate\ninstance positioning and scene composition, and (ii) rendering fine-grained\nattributes using pre-trained ControlNet on any foundational model, without\nadditional training."},{"type":"text","text":" Our 3DIS framework integrates a custom adapter into LDM3D\nfor precise depth-based layouts and employs a finetuning-free method for\nenhanced instance-level attribute rendering. Extensive experiments on\nCOCO-Position and COCO-MIG benchmarks demonstrate that 3DIS significantly\noutperforms existing methods in both layout precision and attribute rendering.\nNotably, 3DIS offers seamless compatibility with diverse foundational models,\nproviding a robust, adaptable solution for advanced multi-instance generation.\nThe code is available at: https://github.com/limuloo/3DIS."}]},{"paper":{"id":"2412.13486","authors":[{"_id":"67828c259a9941c184e76ebd","name":"Zhenhong Sun","hidden":false},{"_id":"67828c259a9941c184e76ebe","name":"Yifu Wang","hidden":false},{"_id":"67828c259a9941c184e76ebf","name":"Yonhon Ng","hidden":false},{"_id":"67828c259a9941c184e76ec0","name":"Yunfei Duan","hidden":false},{"_id":"67828c259a9941c184e76ec1","name":"Daoyi Dong","hidden":false},{"_id":"67828c259a9941c184e76ec2","name":"Hongdong Li","hidden":false},{"_id":"67828c259a9941c184e76ec3","name":"Pan Ji","hidden":false}],"publishedAt":"2024-12-18T04:01:32.000Z","title":"T^3-S2S: Training-free Triplet Tuning for Sketch to Scene Generation","summary":"Scene generation is crucial to many computer graphics applications. Recent\nadvances in generative AI have streamlined sketch-to-image workflows, easing\nthe workload for artists and designers in creating scene concept art. However,\nthese methods often struggle for complex scenes with multiple detailed objects,\nsometimes missing small or uncommon instances. In this paper, we propose a\nTraining-free Triplet Tuning for Sketch-to-Scene (T3-S2S) generation after\nreviewing the entire cross-attention mechanism. This scheme revitalizes the\nexisting ControlNet model, enabling effective handling of multi-instance\ngenerations, involving prompt balance, characteristics prominence, and dense\ntuning. Specifically, this approach enhances keyword representation via the\nprompt balance module, reducing the risk of missing critical instances. It also\nincludes a characteristics prominence module that highlights TopK indices in\neach channel, ensuring essential features are better represented based on token\nsketches. Additionally, it employs dense tuning to refine contour details in\nthe attention map, compensating for instance-related regions. Experiments\nvalidate that our triplet tuning approach substantially improves the\nperformance of existing sketch-to-image models. It consistently generates\ndetailed, multi-instance 2D images, closely adhering to the input prompts and\nenhancing visual quality in complex multi-instance scenes. Code is available at\nhttps://github.com/chaos-sun/t3s2s.git.","upvotes":1,"discussionId":"67828c2b9a9941c184e770d4","githubRepo":"https://github.com/chaos-sun/t3s2s","githubRepoAddedBy":"auto","ai_summary":"The triplet tuning method enhances sketch-to-image generation by balancing prompts, emphasizing characteristics, and refining contours, resulting in detailed and accurate scene images.","ai_keywords":["cross-attention mechanism","Training-free Triplet Tuning","T3-S2S","ControlNet","prompt balance","characteristics prominence","dense tuning","attention map","scene generation","sketch-to-image"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":2},"publishedAt":"2024-12-17T23:01:32.000Z","title":"T^3-S2S: Training-free Triplet Tuning for Sketch to Scene Generation","summary":"Scene generation is crucial to many computer graphics applications. Recent\nadvances in generative AI have streamlined sketch-to-image workflows, easing\nthe workload for artists and designers in creating scene concept art. However,\nthese methods often struggle for complex scenes with multiple detailed objects,\nsometimes missing small or uncommon instances. In this paper, we propose a\nTraining-free Triplet Tuning for Sketch-to-Scene (T3-S2S) generation after\nreviewing the entire cross-attention mechanism. This scheme revitalizes the\nexisting ControlNet model, enabling effective handling of multi-instance\ngenerations, involving prompt balance, characteristics prominence, and dense\ntuning. Specifically, this approach enhances keyword representation via the\nprompt balance module, reducing the risk of missing critical instances. It also\nincludes a characteristics prominence module that highlights TopK indices in\neach channel, ensuring essential features are better represented based on token\nsketches. Additionally, it employs dense tuning to refine contour details in\nthe attention map, compensating for instance-related regions. Experiments\nvalidate that our triplet tuning approach substantially improves the\nperformance of existing sketch-to-image models. It consistently generates\ndetailed, multi-instance 2D images, closely adhering to the input prompts and\nenhancing visual quality in complex multi-instance scenes. Code is available at\nhttps://github.com/chaos-sun/t3s2s.git.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2412.13486.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"T^3-S2S: Training-free Triplet Tuning for Sketch to Scene Generation"}],"highlightedSummary":[{"type":"text","text":"Scene generation is crucial to many computer graphics applications. Recent\nadvances in generative AI have streamlined sketch-to-image workflows, easing\nthe workload for artists and designers in creating scene concept art. "},{"type":"highlight","text":"However,\nthese methods often struggle for complex scenes with multiple detailed objects,\nsometimes missing small or uncommon instances."},{"type":"text","text":" In this paper, we propose a\nTraining-free Triplet Tuning for Sketch-to-Scene (T3-S2S) generation after\nreviewing the entire cross-attention mechanism. This scheme revitalizes the\nexisting ControlNet model, enabling effective handling of multi-instance\ngenerations, involving prompt balance, characteristics prominence, and dense\ntuning. Specifically, this approach enhances keyword representation via the\nprompt balance module, reducing the risk of missing critical instances. It also\nincludes a characteristics prominence module that highlights TopK indices in\neach channel, ensuring essential features are better represented based on token\nsketches. Additionally, it employs dense tuning to refine contour details in\nthe attention map, compensating for instance-related regions. Experiments\nvalidate that our triplet tuning approach substantially improves the\nperformance of existing sketch-to-image models. It consistently generates\ndetailed, multi-instance 2D images, closely adhering to the input prompts and\nenhancing visual quality in complex multi-instance scenes. Code is available at\nhttps://github.com/chaos-sun/t3s2s.git."}]},{"paper":{"id":"2404.19758","authors":[{"_id":"6631c6de6aee5b05e4a0b14d","user":{"_id":"62262603a8218a9b862b18cc","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1648321672399-62262603a8218a9b862b18cc.jpeg","isPro":false,"fullname":"Paul Engstler","user":"paulengstler","type":"user","name":"paulengstler"},"name":"Paul Engstler","status":"claimed_verified","statusLastChangedAt":"2024-05-02T07:10:55.882Z","hidden":false},{"_id":"6631c6de6aee5b05e4a0b14e","name":"Andrea Vedaldi","hidden":false},{"_id":"6631c6de6aee5b05e4a0b14f","name":"Iro Laina","hidden":false},{"_id":"6631c6de6aee5b05e4a0b150","name":"Christian Rupprecht","hidden":false}],"publishedAt":"2024-04-30T17:59:40.000Z","title":"Invisible Stitch: Generating Smooth 3D Scenes with Depth Inpainting","summary":"3D scene generation has quickly become a challenging new research direction,\nfueled by consistent improvements of 2D generative diffusion models. Most prior\nwork in this area generates scenes by iteratively stitching newly generated\nframes with existing geometry. These works often depend on pre-trained\nmonocular depth estimators to lift the generated images into 3D, fusing them\nwith the existing scene representation. These approaches are then often\nevaluated via a text metric, measuring the similarity between the generated\nimages and a given text prompt. In this work, we make two fundamental\ncontributions to the field of 3D scene generation. First, we note that lifting\nimages to 3D with a monocular depth estimation model is suboptimal as it\nignores the geometry of the existing scene. We thus introduce a novel depth\ncompletion model, trained via teacher distillation and self-training to learn\nthe 3D fusion process, resulting in improved geometric coherence of the scene.\nSecond, we introduce a new benchmarking scheme for scene generation methods\nthat is based on ground truth geometry, and thus measures the quality of the\nstructure of the scene.","upvotes":12,"discussionId":"6631c6e06aee5b05e4a0b1cb","githubRepo":"https://github.com/paulengstler/invisible-stitch","githubRepoAddedBy":"auto","ai_summary":"Proposes a new depth completion model and benchmarking scheme for 3D scene generation, improving geometric coherence and structural quality.","ai_keywords":["generative diffusion models","monocular depth estimators","scene generation","depth completion model","teacher distillation","self-training","ground truth geometry"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":185},"publishedAt":"2024-04-30T13:59:40.000Z","title":"Invisible Stitch: Generating Smooth 3D Scenes with Depth Inpainting","summary":"3D scene generation has quickly become a challenging new research direction,\nfueled by consistent improvements of 2D generative diffusion models. Most prior\nwork in this area generates scenes by iteratively stitching newly generated\nframes with existing geometry. These works often depend on pre-trained\nmonocular depth estimators to lift the generated images into 3D, fusing them\nwith the existing scene representation. These approaches are then often\nevaluated via a text metric, measuring the similarity between the generated\nimages and a given text prompt. In this work, we make two fundamental\ncontributions to the field of 3D scene generation. First, we note that lifting\nimages to 3D with a monocular depth estimation model is suboptimal as it\nignores the geometry of the existing scene. We thus introduce a novel depth\ncompletion model, trained via teacher distillation and self-training to learn\nthe 3D fusion process, resulting in improved geometric coherence of the scene.\nSecond, we introduce a new benchmarking scheme for scene generation methods\nthat is based on ground truth geometry, and thus measures the quality of the\nstructure of the scene.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2404.19758.png","numComments":1,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Invisible Stitch: Generating Smooth 3D Scenes with Depth Inpainting"}],"highlightedSummary":[{"type":"highlight","text":"3D scene generation has quickly become a challenging new research direction,\nfueled by consistent improvements of 2D generative diffusion models."},{"type":"text","text":" Most prior\nwork in this area generates scenes by iteratively stitching newly generated\nframes with existing geometry. These works often depend on pre-trained\nmonocular depth estimators to lift the generated images into 3D, fusing them\nwith the existing scene representation. These approaches are then often\nevaluated via a text metric, measuring the similarity between the generated\nimages and a given text prompt. In this work, we make two fundamental\ncontributions to the field of 3D scene generation. First, we note that lifting\nimages to 3D with a monocular depth estimation model is suboptimal as it\nignores the geometry of the existing scene. We thus introduce a novel depth\ncompletion model, trained via teacher distillation and self-training to learn\nthe 3D fusion process, resulting in improved geometric coherence of the scene.\nSecond, we introduce a new benchmarking scheme for scene generation methods\nthat is based on ground truth geometry, and thus measures the quality of the\nstructure of the scene."}]},{"paper":{"id":"2502.00708","authors":[{"_id":"67b9773443d65736d3de62c0","name":"Qixuan Li","hidden":false},{"_id":"67b9773443d65736d3de62c1","name":"Chao Wang","hidden":false},{"_id":"67b9773443d65736d3de62c2","name":"Zongjin He","hidden":false},{"_id":"67b9773443d65736d3de62c3","name":"Yan Peng","hidden":false}],"publishedAt":"2025-02-02T07:47:03.000Z","title":"PhiP-G: Physics-Guided Text-to-3D Compositional Scene Generation","summary":"Text-to-3D asset generation has achieved significant optimization under the\nsupervision of 2D diffusion priors. However, when dealing with compositional\nscenes, existing methods encounter several challenges: 1). failure to ensure\nthat composite scene layouts comply with physical laws; 2). difficulty in\naccurately capturing the assets and relationships described in complex scene\ndescriptions; 3). limited autonomous asset generation capabilities among layout\napproaches leveraging large language models (LLMs). To avoid these compromises,\nwe propose a novel framework for compositional scene generation, PhiP-G, which\nseamlessly integrates generation techniques with layout guidance based on a\nworld model. Leveraging LLM-based agents, PhiP-G analyzes the complex scene\ndescription to generate a scene graph, and integrating a multimodal 2D\ngeneration agent and a 3D Gaussian generation method for targeted assets\ncreation. For the stage of layout, PhiP-G employs a physical pool with adhesion\ncapabilities and a visual supervision agent, forming a world model for layout\nprediction and planning. Extensive experiments demonstrate that PhiP-G\nsignificantly enhances the generation quality and physical rationality of the\ncompositional scenes. Notably, PhiP-G attains state-of-the-art (SOTA)\nperformance in CLIP scores, achieves parity with the leading methods in\ngeneration quality as measured by the T^3Bench, and improves efficiency by\n24x.","upvotes":0,"discussionId":"67b9773543d65736d3de62e1","ai_summary":"PhiP-G framework enhances compositional scene generation by integrating LLM-based agents and multimodal generation methods, achieving state-of-the-art performance with improved efficiency and physical rationality.","ai_keywords":["diffusion priors","compositional scenes","physical laws","scene graph","multimodal generation","3D Gaussian generation","world model","physical pool","adhesion","CLIP scores","T³Bench"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-02-02T02:47:03.000Z","title":"PhiP-G: Physics-Guided Text-to-3D Compositional Scene Generation","summary":"Text-to-3D asset generation has achieved significant optimization under the\nsupervision of 2D diffusion priors. However, when dealing with compositional\nscenes, existing methods encounter several challenges: 1). failure to ensure\nthat composite scene layouts comply with physical laws; 2). difficulty in\naccurately capturing the assets and relationships described in complex scene\ndescriptions; 3). limited autonomous asset generation capabilities among layout\napproaches leveraging large language models (LLMs). To avoid these compromises,\nwe propose a novel framework for compositional scene generation, PhiP-G, which\nseamlessly integrates generation techniques with layout guidance based on a\nworld model. Leveraging LLM-based agents, PhiP-G analyzes the complex scene\ndescription to generate a scene graph, and integrating a multimodal 2D\ngeneration agent and a 3D Gaussian generation method for targeted assets\ncreation. For the stage of layout, PhiP-G employs a physical pool with adhesion\ncapabilities and a visual supervision agent, forming a world model for layout\nprediction and planning. Extensive experiments demonstrate that PhiP-G\nsignificantly enhances the generation quality and physical rationality of the\ncompositional scenes. Notably, PhiP-G attains state-of-the-art (SOTA)\nperformance in CLIP scores, achieves parity with the leading methods in\ngeneration quality as measured by the T^3Bench, and improves efficiency by\n24x.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2502.00708.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"PhiP-G: Physics-Guided Text-to-3D Compositional Scene Generation"}],"highlightedSummary":[{"type":"text","text":"Text-to-3D asset generation has achieved significant optimization under the\nsupervision of 2D diffusion priors. "},{"type":"highlight","text":"However, when dealing with compositional\nscenes, existing methods encounter several challenges: 1)."},{"type":"text","text":" failure to ensure\nthat composite scene layouts comply with physical laws; 2). difficulty in\naccurately capturing the assets and relationships described in complex scene\ndescriptions; 3). limited autonomous asset generation capabilities among layout\napproaches leveraging large language models (LLMs). To avoid these compromises,\nwe propose a novel framework for compositional scene generation, PhiP-G, which\nseamlessly integrates generation techniques with layout guidance based on a\nworld model. Leveraging LLM-based agents, PhiP-G analyzes the complex scene\ndescription to generate a scene graph, and integrating a multimodal 2D\ngeneration agent and a 3D Gaussian generation method for targeted assets\ncreation. For the stage of layout, PhiP-G employs a physical pool with adhesion\ncapabilities and a visual supervision agent, forming a world model for layout\nprediction and planning. Extensive experiments demonstrate that PhiP-G\nsignificantly enhances the generation quality and physical rationality of the\ncompositional scenes. Notably, PhiP-G attains state-of-the-art (SOTA)\nperformance in CLIP scores, achieves parity with the leading methods in\ngeneration quality as measured by the T^3Bench, and improves efficiency by\n24x."}]},{"paper":{"id":"2504.11734","authors":[{"_id":"6825c174ee98cb4abe4f3f2b","name":"Xiang Tang","hidden":false},{"_id":"6825c174ee98cb4abe4f3f2c","name":"Ruotong Li","hidden":false},{"_id":"6825c174ee98cb4abe4f3f2d","name":"Xiaopeng Fan","hidden":false}],"publishedAt":"2025-04-16T03:22:06.000Z","title":"Recent Advance in 3D Object and Scene Generation: A Survey","summary":"In recent years, the demand for 3D content has grown exponentially with\nintelligent upgrading of interactive media, extended reality (XR), and\nMetaverse industries. In order to overcome the limitation of traditional manual\nmodeling approaches, such as labor-intensive workflows and prolonged production\ncycles, revolutionary advances have been achieved through the convergence of\nnovel 3D representation paradigms and artificial intelligence generative\ntechnologies. In this survey, we conduct a systematically review of the\ncutting-edge achievements in static 3D object and scene generation, as well as\nestablish a comprehensive technical framework through systematic\ncategorization. Specifically, we initiate our analysis with mainstream 3D\nobject representations, followed by in-depth exploration of two principal\ntechnical pathways in object generation: data-driven supervised learning\nmethods and deep generative model-based approaches. Regarding scene generation,\nwe focus on three dominant paradigms: layout-guided compositional synthesis, 2D\nprior-based scene generation, and rule-driven modeling. Finally, we critically\nexamine persistent challenges in 3D generation and propose potential research\ndirections for future investigation. This survey aims to provide readers with a\nstructured understanding of state-of-the-art 3D generation technologies while\ninspiring researchers to undertake more exploration in this domain.","upvotes":0,"discussionId":"6825c175ee98cb4abe4f3f5e","ai_summary":"We review state-of-the-art 3D object and scene generation technologies, focusing on data-driven and deep generative methods, while identifying challenges and suggesting future research directions.","ai_keywords":["deep generative models","layout-guided compositional synthesis","2D prior-based scene generation","rule-driven modeling"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-04-15T23:22:06.000Z","title":"Recent Advance in 3D Object and Scene Generation: A Survey","summary":"In recent years, the demand for 3D content has grown exponentially with\nintelligent upgrading of interactive media, extended reality (XR), and\nMetaverse industries. In order to overcome the limitation of traditional manual\nmodeling approaches, such as labor-intensive workflows and prolonged production\ncycles, revolutionary advances have been achieved through the convergence of\nnovel 3D representation paradigms and artificial intelligence generative\ntechnologies. In this survey, we conduct a systematically review of the\ncutting-edge achievements in static 3D object and scene generation, as well as\nestablish a comprehensive technical framework through systematic\ncategorization. Specifically, we initiate our analysis with mainstream 3D\nobject representations, followed by in-depth exploration of two principal\ntechnical pathways in object generation: data-driven supervised learning\nmethods and deep generative model-based approaches. Regarding scene generation,\nwe focus on three dominant paradigms: layout-guided compositional synthesis, 2D\nprior-based scene generation, and rule-driven modeling. Finally, we critically\nexamine persistent challenges in 3D generation and propose potential research\ndirections for future investigation. This survey aims to provide readers with a\nstructured understanding of state-of-the-art 3D generation technologies while\ninspiring researchers to undertake more exploration in this domain.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2504.11734.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Recent Advance in 3D Object and Scene Generation: A Survey"}],"highlightedSummary":[{"type":"text","text":"In recent years, the demand for 3D content has grown exponentially with\nintelligent upgrading of interactive media, extended reality (XR), and\nMetaverse industries. In order to overcome the limitation of traditional manual\nmodeling approaches, such as labor-intensive workflows and prolonged production\ncycles, revolutionary advances have been achieved through the convergence of\nnovel 3D representation paradigms and artificial intelligence generative\ntechnologies. In this survey, we conduct a systematically review of the\ncutting-edge achievements in static 3D object and scene generation, as well as\nestablish a comprehensive technical framework through systematic\ncategorization. Specifically, we initiate our analysis with mainstream 3D\nobject representations, followed by in-depth exploration of two principal\ntechnical pathways in object generation: data-driven supervised learning\nmethods and deep generative model-based approaches. "},{"type":"highlight","text":"Regarding scene generation,\nwe focus on three dominant paradigms: layout-guided compositional synthesis, 2D\nprior-based scene generation, and rule-driven modeling."},{"type":"text","text":" Finally, we critically\nexamine persistent challenges in 3D generation and propose potential research\ndirections for future investigation. This survey aims to provide readers with a\nstructured understanding of state-of-the-art 3D generation technologies while\ninspiring researchers to undertake more exploration in this domain."}]},{"paper":{"id":"2403.16993","authors":[{"_id":"660394d69ad3ee7b9488b02c","user":{"_id":"62f687a0c58915315c4ff75d","avatarUrl":"/avatars/b657180c7666735062782edd4f6a69c9.svg","isPro":false,"fullname":"Dejia Xu","user":"ir1d","type":"user","name":"ir1d"},"name":"Dejia Xu","status":"claimed_verified","statusLastChangedAt":"2024-06-03T07:05:56.135Z","hidden":false},{"_id":"660394d69ad3ee7b9488b02d","user":{"_id":"64cb2227e984d09bed90fdc5","avatarUrl":"/avatars/15b04cfbd935bd01c84020c8a8e6c8c3.svg","isPro":false,"fullname":"Hanwen Liang","user":"hw-liang","type":"user","name":"hw-liang"},"name":"Hanwen Liang","status":"claimed_verified","statusLastChangedAt":"2024-12-18T09:55:11.662Z","hidden":false},{"_id":"660394d69ad3ee7b9488b02e","user":{"_id":"65915eab5b7553ca5c9a57d2","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/65915eab5b7553ca5c9a57d2/ZWhWeste-w0HKEjFssl09.jpeg","isPro":false,"fullname":"Neel Bhatt","user":"neel1302","type":"user","name":"neel1302"},"name":"Neel P. Bhatt","status":"claimed_verified","statusLastChangedAt":"2024-07-08T08:51:30.065Z","hidden":false},{"_id":"660394d69ad3ee7b9488b02f","name":"Hezhen Hu","hidden":false},{"_id":"660394d69ad3ee7b9488b030","name":"Hanxue Liang","hidden":false},{"_id":"660394d69ad3ee7b9488b031","name":"Konstantinos N. Plataniotis","hidden":false},{"_id":"660394d69ad3ee7b9488b032","name":"Zhangyang Wang","hidden":false}],"publishedAt":"2024-03-25T17:55:52.000Z","title":"Comp4D: LLM-Guided Compositional 4D Scene Generation","summary":"Recent advancements in diffusion models for 2D and 3D content creation have\nsparked a surge of interest in generating 4D content. However, the scarcity of\n3D scene datasets constrains current methodologies to primarily object-centric\ngeneration. To overcome this limitation, we present Comp4D, a novel framework\nfor Compositional 4D Generation. Unlike conventional methods that generate a\nsingular 4D representation of the entire scene, Comp4D innovatively constructs\neach 4D object within the scene separately. Utilizing Large Language Models\n(LLMs), the framework begins by decomposing an input text prompt into distinct\nentities and maps out their trajectories. It then constructs the compositional\n4D scene by accurately positioning these objects along their designated paths.\nTo refine the scene, our method employs a compositional score distillation\ntechnique guided by the pre-defined trajectories, utilizing pre-trained\ndiffusion models across text-to-image, text-to-video, and text-to-3D domains.\nExtensive experiments demonstrate our outstanding 4D content creation\ncapability compared to prior arts, showcasing superior visual quality, motion\nfidelity, and enhanced object interactions.","upvotes":3,"discussionId":"660394d79ad3ee7b9488b05a","ai_summary":"A novel framework for 4D content creation, Comp4D, uses LLMs and compositional score distillation with pre-trained diffusion models to generate high-quality, motion-fidelity 4D scenes.","ai_keywords":["diffusion models","4D content creation","Comp4D","compositional generation","Large Language Models (LLMs)","score distillation","text-to-image","text-to-video","text-to-3D","visual quality","motion fidelity","object interactions"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-03-25T13:55:52.000Z","title":"Comp4D: LLM-Guided Compositional 4D Scene Generation","summary":"Recent advancements in diffusion models for 2D and 3D content creation have\nsparked a surge of interest in generating 4D content. However, the scarcity of\n3D scene datasets constrains current methodologies to primarily object-centric\ngeneration. To overcome this limitation, we present Comp4D, a novel framework\nfor Compositional 4D Generation. Unlike conventional methods that generate a\nsingular 4D representation of the entire scene, Comp4D innovatively constructs\neach 4D object within the scene separately. Utilizing Large Language Models\n(LLMs), the framework begins by decomposing an input text prompt into distinct\nentities and maps out their trajectories. It then constructs the compositional\n4D scene by accurately positioning these objects along their designated paths.\nTo refine the scene, our method employs a compositional score distillation\ntechnique guided by the pre-defined trajectories, utilizing pre-trained\ndiffusion models across text-to-image, text-to-video, and text-to-3D domains.\nExtensive experiments demonstrate our outstanding 4D content creation\ncapability compared to prior arts, showcasing superior visual quality, motion\nfidelity, and enhanced object interactions.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2403.16993.png","numComments":1,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Comp4D: LLM-Guided Compositional 4D Scene Generation"}],"highlightedSummary":[{"type":"text","text":"Recent advancements in diffusion models for 2D and 3D content creation have\nsparked a surge of interest in generating 4D content. However, the scarcity of\n3D scene datasets constrains current methodologies to primarily object-centric\ngeneration. "},{"type":"highlight","text":"To overcome this limitation, we present Comp4D, a novel framework\nfor Compositional 4D Generation."},{"type":"text","text":" Unlike conventional methods that generate a\nsingular 4D representation of the entire scene, Comp4D innovatively constructs\neach 4D object within the scene separately. Utilizing Large Language Models\n(LLMs), the framework begins by decomposing an input text prompt into distinct\nentities and maps out their trajectories. It then constructs the compositional\n4D scene by accurately positioning these objects along their designated paths.\nTo refine the scene, our method employs a compositional score distillation\ntechnique guided by the pre-defined trajectories, utilizing pre-trained\ndiffusion models across text-to-image, text-to-video, and text-to-3D domains.\nExtensive experiments demonstrate our outstanding 4D content creation\ncapability compared to prior arts, showcasing superior visual quality, motion\nfidelity, and enhanced object interactions."}]},{"paper":{"id":"2304.12670","authors":[{"_id":"6505ce8f4fd1985c63fc2d43","user":{"_id":"63b53090889aa6707f07afd6","avatarUrl":"/avatars/595ecfea802c6faed50e031d442f1639.svg","isPro":false,"fullname":"weiyuli","user":"wyysf","type":"user","name":"wyysf"},"name":"Weiyu Li","status":"claimed_verified","statusLastChangedAt":"2023-10-24T07:30:15.355Z","hidden":false},{"_id":"6505ce8f4fd1985c63fc2d44","name":"Xuelin Chen","hidden":false},{"_id":"6505ce8f4fd1985c63fc2d45","name":"Jue Wang","hidden":false},{"_id":"6505ce8f4fd1985c63fc2d46","name":"Baoquan Chen","hidden":false}],"publishedAt":"2023-04-25T09:19:11.000Z","title":"Patch-based 3D Natural Scene Generation from a Single Example","summary":"We target a 3D generative model for general natural scenes that are typically\nunique and intricate. Lacking the necessary volumes of training data, along\nwith the difficulties of having ad hoc designs in presence of varying scene\ncharacteristics, renders existing setups intractable. Inspired by classical\npatch-based image models, we advocate for synthesizing 3D scenes at the patch\nlevel, given a single example. At the core of this work lies important\nalgorithmic designs w.r.t the scene representation and generative patch\nnearest-neighbor module, that address unique challenges arising from lifting\nclassical 2D patch-based framework to 3D generation. These design choices, on a\ncollective level, contribute to a robust, effective, and efficient model that\ncan generate high-quality general natural scenes with both realistic geometric\nstructure and visual appearance, in large quantities and varieties, as\ndemonstrated upon a variety of exemplar scenes.","upvotes":1,"discussionId":"6505ce994fd1985c63fc304e","ai_summary":"A patch-based approach for 3D scene generation from a single example addresses unique challenges in 3D modeling, resulting in a robust model capable of producing high-quality natural scenes.","ai_keywords":["3D generative model","patch-based image models","scene representation","generative patch nearest-neighbor module"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2023-04-25T05:19:11.000Z","title":"Patch-based 3D Natural Scene Generation from a Single Example","summary":"We target a 3D generative model for general natural scenes that are typically\nunique and intricate. Lacking the necessary volumes of training data, along\nwith the difficulties of having ad hoc designs in presence of varying scene\ncharacteristics, renders existing setups intractable. Inspired by classical\npatch-based image models, we advocate for synthesizing 3D scenes at the patch\nlevel, given a single example. At the core of this work lies important\nalgorithmic designs w.r.t the scene representation and generative patch\nnearest-neighbor module, that address unique challenges arising from lifting\nclassical 2D patch-based framework to 3D generation. These design choices, on a\ncollective level, contribute to a robust, effective, and efficient model that\ncan generate high-quality general natural scenes with both realistic geometric\nstructure and visual appearance, in large quantities and varieties, as\ndemonstrated upon a variety of exemplar scenes.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2304.12670.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Patch-based 3D Natural Scene Generation from a Single Example"}],"highlightedSummary":[{"type":"text","text":"We target a 3D generative model for general natural scenes that are typically\nunique and intricate. Lacking the necessary volumes of training data, along\nwith the difficulties of having ad hoc designs in presence of varying scene\ncharacteristics, renders existing setups intractable. Inspired by classical\npatch-based image models, we advocate for synthesizing 3D scenes at the patch\nlevel, given a single example. At the core of this work lies important\nalgorithmic designs w.r.t the scene representation and generative patch\nnearest-neighbor module, that address unique challenges arising from lifting\nclassical 2D patch-based framework to 3D generation. These design choices, on a\ncollective level, contribute to a robust, effective, and efficient model that\ncan generate high-quality general natural scenes with both realistic geometric\nstructure and visual appearance, in large quantities and varieties, as\ndemonstrated upon a variety of exemplar scenes."}]},{"paper":{"id":"2410.09009","authors":[{"_id":"670c7992f6b3c3ad2cbe2f2b","user":{"_id":"64fde4e252e82dd432b74ce9","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64fde4e252e82dd432b74ce9/-CQZbBP7FsPPyawYrsi4z.jpeg","isPro":false,"fullname":"Ling Yang","user":"Lingaaaaaaa","type":"user","name":"Lingaaaaaaa"},"name":"Ling Yang","status":"admin_assigned","statusLastChangedAt":"2024-10-14T12:35:15.951Z","hidden":false},{"_id":"670c7992f6b3c3ad2cbe2f2c","name":"Zixiang Zhang","hidden":false},{"_id":"670c7992f6b3c3ad2cbe2f2d","name":"Junlin Han","hidden":false},{"_id":"670c7992f6b3c3ad2cbe2f2e","user":{"_id":"6671214c92412fd4640714eb","avatarUrl":"/avatars/48fa84e7bc3bb92ad0192aa26b32de10.svg","isPro":false,"fullname":"Bohan Zeng","user":"zbhpku","type":"user","name":"zbhpku"},"name":"Bohan Zeng","status":"admin_assigned","statusLastChangedAt":"2024-10-14T12:30:09.672Z","hidden":false},{"_id":"670c7992f6b3c3ad2cbe2f2f","user":{"_id":"638e29cf319f9c746b87ad4b","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/638e29cf319f9c746b87ad4b/YjKrcY-NyvETSWrRnC0Tx.jpeg","isPro":false,"fullname":"Runjia Li","user":"liguang0115","type":"user","name":"liguang0115"},"name":"Runjia Li","status":"admin_assigned","statusLastChangedAt":"2024-10-14T12:29:45.711Z","hidden":false},{"_id":"670c7992f6b3c3ad2cbe2f30","user":{"_id":"6565ed28a5ec0231cb07225f","avatarUrl":"/avatars/7f95bba9aa7811d56eecb380827abfac.svg","isPro":false,"fullname":"prof philip torr","user":"philiptorr","type":"user","name":"philiptorr"},"name":"Philip Torr","status":"admin_assigned","statusLastChangedAt":"2024-10-14T12:23:22.107Z","hidden":false},{"_id":"670c7992f6b3c3ad2cbe2f31","name":"Wentao Zhang","hidden":false}],"publishedAt":"2024-10-11T17:26:00.000Z","title":"Semantic Score Distillation Sampling for Compositional Text-to-3D\n  Generation","summary":"Generating high-quality 3D assets from textual descriptions remains a pivotal\nchallenge in computer graphics and vision research. Due to the scarcity of 3D\ndata, state-of-the-art approaches utilize pre-trained 2D diffusion priors,\noptimized through Score Distillation Sampling (SDS). Despite progress, crafting\ncomplex 3D scenes featuring multiple objects or intricate interactions is still\ndifficult. To tackle this, recent methods have incorporated box or layout\nguidance. However, these layout-guided compositional methods often struggle to\nprovide fine-grained control, as they are generally coarse and lack\nexpressiveness. To overcome these challenges, we introduce a novel SDS\napproach, Semantic Score Distillation Sampling (SemanticSDS), designed to\neffectively improve the expressiveness and accuracy of compositional text-to-3D\ngeneration. Our approach integrates new semantic embeddings that maintain\nconsistency across different rendering views and clearly differentiate between\nvarious objects and parts. These embeddings are transformed into a semantic\nmap, which directs a region-specific SDS process, enabling precise optimization\nand compositional generation. By leveraging explicit semantic guidance, our\nmethod unlocks the compositional capabilities of existing pre-trained diffusion\nmodels, thereby achieving superior quality in 3D content generation,\nparticularly for complex objects and scenes. Experimental results demonstrate\nthat our SemanticSDS framework is highly effective for generating\nstate-of-the-art complex 3D content. Code:\nhttps://github.com/YangLing0818/SemanticSDS-3D","upvotes":15,"discussionId":"670c7994f6b3c3ad2cbe3018","ai_summary":"A new Semantic Score Distillation Sampling approach improves accuracy and expressiveness in text-to-3D generation by using semantic embeddings and maps, enhancing pre-trained diffusion models.","ai_keywords":["Score Distillation Sampling","Semantic Score Distillation Sampling","semantic embeddings","semantic map","region-specific SDS","pre-trained diffusion models"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-10-11T13:26:00.000Z","title":"Semantic Score Distillation Sampling for Compositional Text-to-3D\n  Generation","summary":"Generating high-quality 3D assets from textual descriptions remains a pivotal\nchallenge in computer graphics and vision research. Due to the scarcity of 3D\ndata, state-of-the-art approaches utilize pre-trained 2D diffusion priors,\noptimized through Score Distillation Sampling (SDS). Despite progress, crafting\ncomplex 3D scenes featuring multiple objects or intricate interactions is still\ndifficult. To tackle this, recent methods have incorporated box or layout\nguidance. However, these layout-guided compositional methods often struggle to\nprovide fine-grained control, as they are generally coarse and lack\nexpressiveness. To overcome these challenges, we introduce a novel SDS\napproach, Semantic Score Distillation Sampling (SemanticSDS), designed to\neffectively improve the expressiveness and accuracy of compositional text-to-3D\ngeneration. Our approach integrates new semantic embeddings that maintain\nconsistency across different rendering views and clearly differentiate between\nvarious objects and parts. These embeddings are transformed into a semantic\nmap, which directs a region-specific SDS process, enabling precise optimization\nand compositional generation. By leveraging explicit semantic guidance, our\nmethod unlocks the compositional capabilities of existing pre-trained diffusion\nmodels, thereby achieving superior quality in 3D content generation,\nparticularly for complex objects and scenes. Experimental results demonstrate\nthat our SemanticSDS framework is highly effective for generating\nstate-of-the-art complex 3D content. Code:\nhttps://github.com/YangLing0818/SemanticSDS-3D","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2410.09009.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Semantic Score Distillation Sampling for Compositional Text-to-3D\n  Generation"}],"highlightedSummary":[{"type":"text","text":"Generating high-quality 3D assets from textual descriptions remains a pivotal\nchallenge in computer graphics and vision research. Due to the scarcity of 3D\ndata, state-of-the-art approaches utilize pre-trained 2D diffusion priors,\noptimized through Score Distillation Sampling (SDS). "},{"type":"highlight","text":"Despite progress, crafting\ncomplex 3D scenes featuring multiple objects or intricate interactions is still\ndifficult."},{"type":"text","text":" To tackle this, recent methods have incorporated box or layout\nguidance. However, these layout-guided compositional methods often struggle to\nprovide fine-grained control, as they are generally coarse and lack\nexpressiveness. To overcome these challenges, we introduce a novel SDS\napproach, Semantic Score Distillation Sampling (SemanticSDS), designed to\neffectively improve the expressiveness and accuracy of compositional text-to-3D\ngeneration. Our approach integrates new semantic embeddings that maintain\nconsistency across different rendering views and clearly differentiate between\nvarious objects and parts. These embeddings are transformed into a semantic\nmap, which directs a region-specific SDS process, enabling precise optimization\nand compositional generation. By leveraging explicit semantic guidance, our\nmethod unlocks the compositional capabilities of existing pre-trained diffusion\nmodels, thereby achieving superior quality in 3D content generation,\nparticularly for complex objects and scenes. Experimental results demonstrate\nthat our SemanticSDS framework is highly effective for generating\nstate-of-the-art complex 3D content. Code:\nhttps://github.com/YangLing0818/SemanticSDS-3D"}]},{"paper":{"id":"2502.01949","authors":[{"_id":"67bd870e0417e7f92292ecc4","name":"Yang Zhou","hidden":false},{"_id":"67bd870e0417e7f92292ecc5","name":"Zongjin He","hidden":false},{"_id":"67bd870e0417e7f92292ecc6","name":"Qixuan Li","hidden":false},{"_id":"67bd870e0417e7f92292ecc7","name":"Chao Wang","hidden":false}],"publishedAt":"2025-02-04T02:51:37.000Z","title":"LAYOUTDREAMER: Physics-guided Layout for Text-to-3D Compositional Scene\n  Generation","summary":"Recently, the field of text-guided 3D scene generation has garnered\nsignificant attention. High-quality generation that aligns with physical\nrealism and high controllability is crucial for practical 3D scene\napplications. However, existing methods face fundamental limitations: (i)\ndifficulty capturing complex relationships between multiple objects described\nin the text, (ii) inability to generate physically plausible scene layouts, and\n(iii) lack of controllability and extensibility in compositional scenes. In\nthis paper, we introduce LayoutDreamer, a framework that leverages 3D Gaussian\nSplatting (3DGS) to facilitate high-quality, physically consistent\ncompositional scene generation guided by text. Specifically, given a text\nprompt, we convert it into a directed scene graph and adaptively adjust the\ndensity and layout of the initial compositional 3D Gaussians. Subsequently,\ndynamic camera adjustments are made based on the training focal point to ensure\nentity-level generation quality. Finally, by extracting directed dependencies\nfrom the scene graph, we tailor physical and layout energy to ensure both\nrealism and flexibility. Comprehensive experiments demonstrate that\nLayoutDreamer outperforms other compositional scene generation quality and\nsemantic alignment methods. Specifically, it achieves state-of-the-art (SOTA)\nperformance in the multiple objects generation metric of T3Bench.","upvotes":1,"discussionId":"67bd87100417e7f92292ed6d","ai_summary":"LayoutDreamer uses 3D Gaussian Splatting for high-quality, physically consistent 3D scene generation guided by text, outperforming existing methods in multiple objects generation.","ai_keywords":["3D Gaussian Splatting","directed scene graph","adaptive density","layout","dynamic camera adjustments","entity-level generation","physical energy","layout energy","T3Bench","multiple objects generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-02-03T21:51:37.000Z","title":"LAYOUTDREAMER: Physics-guided Layout for Text-to-3D Compositional Scene\n  Generation","summary":"Recently, the field of text-guided 3D scene generation has garnered\nsignificant attention. High-quality generation that aligns with physical\nrealism and high controllability is crucial for practical 3D scene\napplications. However, existing methods face fundamental limitations: (i)\ndifficulty capturing complex relationships between multiple objects described\nin the text, (ii) inability to generate physically plausible scene layouts, and\n(iii) lack of controllability and extensibility in compositional scenes. In\nthis paper, we introduce LayoutDreamer, a framework that leverages 3D Gaussian\nSplatting (3DGS) to facilitate high-quality, physically consistent\ncompositional scene generation guided by text. Specifically, given a text\nprompt, we convert it into a directed scene graph and adaptively adjust the\ndensity and layout of the initial compositional 3D Gaussians. Subsequently,\ndynamic camera adjustments are made based on the training focal point to ensure\nentity-level generation quality. Finally, by extracting directed dependencies\nfrom the scene graph, we tailor physical and layout energy to ensure both\nrealism and flexibility. Comprehensive experiments demonstrate that\nLayoutDreamer outperforms other compositional scene generation quality and\nsemantic alignment methods. Specifically, it achieves state-of-the-art (SOTA)\nperformance in the multiple objects generation metric of T3Bench.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2502.01949.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"LAYOUTDREAMER: Physics-guided Layout for Text-to-3D Compositional Scene\n  Generation"}],"highlightedSummary":[{"type":"text","text":"Recently, the field of text-guided 3D scene generation has garnered\nsignificant attention. High-quality generation that aligns with physical\nrealism and high controllability is crucial for practical 3D scene\napplications. "},{"type":"highlight","text":"However, existing methods face fundamental limitations: (i)\ndifficulty capturing complex relationships between multiple objects described\nin the text, (ii) inability to generate physically plausible scene layouts, and\n(iii) lack of controllability and extensibility in compositional scenes."},{"type":"text","text":" In\nthis paper, we introduce LayoutDreamer, a framework that leverages 3D Gaussian\nSplatting (3DGS) to facilitate high-quality, physically consistent\ncompositional scene generation guided by text. Specifically, given a text\nprompt, we convert it into a directed scene graph and adaptively adjust the\ndensity and layout of the initial compositional 3D Gaussians. Subsequently,\ndynamic camera adjustments are made based on the training focal point to ensure\nentity-level generation quality. Finally, by extracting directed dependencies\nfrom the scene graph, we tailor physical and layout energy to ensure both\nrealism and flexibility. Comprehensive experiments demonstrate that\nLayoutDreamer outperforms other compositional scene generation quality and\nsemantic alignment methods. Specifically, it achieves state-of-the-art (SOTA)\nperformance in the multiple objects generation metric of T3Bench."}]},{"paper":{"id":"2408.05477","authors":[{"_id":"66d1926ccd11d475b46f2a26","user":{"_id":"67da6acc05101e8e1d2c20a2","avatarUrl":"/avatars/1cfa3a1f59687db58af4e1b4a8767bfd.svg","isPro":false,"fullname":"Yang","user":"Yiying12","type":"user","name":"Yiying12"},"name":"Yiying Yang","status":"claimed_verified","statusLastChangedAt":"2025-04-11T12:02:15.680Z","hidden":false},{"_id":"66d1926ccd11d475b46f2a27","name":"Fukun Yin","hidden":false},{"_id":"66d1926ccd11d475b46f2a28","name":"Jiayuan Fan","hidden":false},{"_id":"66d1926ccd11d475b46f2a29","name":"Xin Chen","hidden":false},{"_id":"66d1926ccd11d475b46f2a2a","name":"Wanzhang Li","hidden":false},{"_id":"66d1926ccd11d475b46f2a2b","name":"Gang Yu","hidden":false}],"publishedAt":"2024-08-10T08:09:57.000Z","title":"Scene123: One Prompt to 3D Scene Generation via Video-Assisted and\n  Consistency-Enhanced MAE","summary":"As Artificial Intelligence Generated Content (AIGC) advances, a variety of\nmethods have been developed to generate text, images, videos, and 3D objects\nfrom single or multimodal inputs, contributing efforts to emulate human-like\ncognitive content creation. However, generating realistic large-scale scenes\nfrom a single input presents a challenge due to the complexities involved in\nensuring consistency across extrapolated views generated by models. Benefiting\nfrom recent video generation models and implicit neural representations, we\npropose Scene123, a 3D scene generation model, that not only ensures realism\nand diversity through the video generation framework but also uses implicit\nneural fields combined with Masked Autoencoders (MAE) to effectively ensures\nthe consistency of unseen areas across views. Specifically, we initially warp\nthe input image (or an image generated from text) to simulate adjacent views,\nfilling the invisible areas with the MAE model. However, these filled images\nusually fail to maintain view consistency, thus we utilize the produced views\nto optimize a neural radiance field, enhancing geometric consistency.\n  Moreover, to further enhance the details and texture fidelity of generated\nviews, we employ a GAN-based Loss against images derived from the input image\nthrough the video generation model. Extensive experiments demonstrate that our\nmethod can generate realistic and consistent scenes from a single prompt. Both\nqualitative and quantitative results indicate that our approach surpasses\nexisting state-of-the-art methods. We show encourage video examples at\nhttps://yiyingyang12.github.io/Scene123.github.io/.","upvotes":1,"discussionId":"66d19272cd11d475b46f2bb2","ai_summary":"Scene123 enhances 3D scene generation from a single input using implicit neural fields, Masked Autoencoders, and a GAN-based loss to ensure realism and consistency across views.","ai_keywords":["implicit neural representations","Masked Autoencoders","neural radiance field","GAN-based Loss"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-08-10T04:09:57.000Z","title":"Scene123: One Prompt to 3D Scene Generation via Video-Assisted and\n  Consistency-Enhanced MAE","summary":"As Artificial Intelligence Generated Content (AIGC) advances, a variety of\nmethods have been developed to generate text, images, videos, and 3D objects\nfrom single or multimodal inputs, contributing efforts to emulate human-like\ncognitive content creation. However, generating realistic large-scale scenes\nfrom a single input presents a challenge due to the complexities involved in\nensuring consistency across extrapolated views generated by models. Benefiting\nfrom recent video generation models and implicit neural representations, we\npropose Scene123, a 3D scene generation model, that not only ensures realism\nand diversity through the video generation framework but also uses implicit\nneural fields combined with Masked Autoencoders (MAE) to effectively ensures\nthe consistency of unseen areas across views. Specifically, we initially warp\nthe input image (or an image generated from text) to simulate adjacent views,\nfilling the invisible areas with the MAE model. However, these filled images\nusually fail to maintain view consistency, thus we utilize the produced views\nto optimize a neural radiance field, enhancing geometric consistency.\n  Moreover, to further enhance the details and texture fidelity of generated\nviews, we employ a GAN-based Loss against images derived from the input image\nthrough the video generation model. Extensive experiments demonstrate that our\nmethod can generate realistic and consistent scenes from a single prompt. Both\nqualitative and quantitative results indicate that our approach surpasses\nexisting state-of-the-art methods. We show encourage video examples at\nhttps://yiyingyang12.github.io/Scene123.github.io/.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2408.05477.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Scene123: One Prompt to 3D Scene Generation via Video-Assisted and\n  Consistency-Enhanced MAE"}],"highlightedSummary":[{"type":"text","text":"As Artificial Intelligence Generated Content (AIGC) advances, a variety of\nmethods have been developed to generate text, images, videos, and 3D objects\nfrom single or multimodal inputs, contributing efforts to emulate human-like\ncognitive content creation. However, generating realistic large-scale scenes\nfrom a single input presents a challenge due to the complexities involved in\nensuring consistency across extrapolated views generated by models. Benefiting\nfrom recent video generation models and implicit neural representations, we\npropose Scene123, a 3D scene generation model, that not only ensures realism\nand diversity through the video generation framework but also uses implicit\nneural fields combined with Masked Autoencoders (MAE) to effectively ensures\nthe consistency of unseen areas across views. Specifically, we initially warp\nthe input image (or an image generated from text) to simulate adjacent views,\nfilling the invisible areas with the MAE model. However, these filled images\nusually fail to maintain view consistency, thus we utilize the produced views\nto optimize a neural radiance field, enhancing geometric consistency.\n  Moreover, to further enhance the details and texture fidelity of generated\nviews, we employ a GAN-based Loss against images derived from the input image\nthrough the video generation model. "},{"type":"highlight","text":"Extensive experiments demonstrate that our\nmethod can generate realistic and consistent scenes from a single prompt."},{"type":"text","text":" Both\nqualitative and quantitative results indicate that our approach surpasses\nexisting state-of-the-art methods. We show encourage video examples at\nhttps://yiyingyang12.github.io/Scene123.github.io/."}]},{"paper":{"id":"2007.11744","authors":[{"_id":"6675d02c52facbe19b0b7af5","user":{"_id":"64b6ce23dbbd1f2cdb624d56","avatarUrl":"/avatars/022738ce8f76fa9545b3e363d7264b53.svg","isPro":false,"fullname":"Andrew Luo","user":"aluo-x","type":"user","name":"aluo-x"},"name":"Andrew Luo","status":"claimed_verified","statusLastChangedAt":"2024-06-24T07:30:45.341Z","hidden":false},{"_id":"6675d02c52facbe19b0b7af6","name":"Zhoutong Zhang","hidden":false},{"_id":"6675d02c52facbe19b0b7af7","name":"Jiajun Wu","hidden":false},{"_id":"6675d02c52facbe19b0b7af8","name":"Joshua B. Tenenbaum","hidden":false}],"publishedAt":"2020-07-23T01:35:55.000Z","title":"End-to-End Optimization of Scene Layout","summary":"We propose an end-to-end variational generative model for scene layout\nsynthesis conditioned on scene graphs. Unlike unconditional scene layout\ngeneration, we use scene graphs as an abstract but general representation to\nguide the synthesis of diverse scene layouts that satisfy relationships\nincluded in the scene graph. This gives rise to more flexible control over the\nsynthesis process, allowing various forms of inputs such as scene layouts\nextracted from sentences or inferred from a single color image. Using our\nconditional layout synthesizer, we can generate various layouts that share the\nsame structure of the input example. In addition to this conditional generation\ndesign, we also integrate a differentiable rendering module that enables layout\nrefinement using only 2D projections of the scene. Given a depth and a\nsemantics map, the differentiable rendering module enables optimizing over the\nsynthesized layout to fit the given input in an analysis-by-synthesis fashion.\nExperiments suggest that our model achieves higher accuracy and diversity in\nconditional scene synthesis and allows exemplar-based scene generation from\nvarious input forms.","upvotes":0,"discussionId":"6675d02d52facbe19b0b7b4f","githubRepo":"https://github.com/aluo-x/3D_SLN","githubRepoAddedBy":"auto","ai_summary":"A variational generative model synthesizes scene layouts from scene graphs, allowing flexible control and integration with differentiable rendering for layout refinement.","ai_keywords":["variational generative model","scene layout synthesis","scene graphs","conditional layout synthesizer","differentiable rendering module","analysis-by-synthesis","exemplar-based scene generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":56},"publishedAt":"2020-07-22T21:35:55.000Z","title":"End-to-End Optimization of Scene Layout","summary":"We propose an end-to-end variational generative model for scene layout\nsynthesis conditioned on scene graphs. Unlike unconditional scene layout\ngeneration, we use scene graphs as an abstract but general representation to\nguide the synthesis of diverse scene layouts that satisfy relationships\nincluded in the scene graph. This gives rise to more flexible control over the\nsynthesis process, allowing various forms of inputs such as scene layouts\nextracted from sentences or inferred from a single color image. Using our\nconditional layout synthesizer, we can generate various layouts that share the\nsame structure of the input example. In addition to this conditional generation\ndesign, we also integrate a differentiable rendering module that enables layout\nrefinement using only 2D projections of the scene. Given a depth and a\nsemantics map, the differentiable rendering module enables optimizing over the\nsynthesized layout to fit the given input in an analysis-by-synthesis fashion.\nExperiments suggest that our model achieves higher accuracy and diversity in\nconditional scene synthesis and allows exemplar-based scene generation from\nvarious input forms.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2007.11744.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"End-to-End Optimization of Scene Layout"}],"highlightedSummary":[{"type":"text","text":"We propose an end-to-end variational generative model for scene layout\nsynthesis conditioned on scene graphs. Unlike unconditional scene layout\ngeneration, we use scene graphs as an abstract but general representation to\nguide the synthesis of diverse scene layouts that satisfy relationships\nincluded in the scene graph. This gives rise to more flexible control over the\nsynthesis process, allowing various forms of inputs such as scene layouts\nextracted from sentences or inferred from a single color image. Using our\nconditional layout synthesizer, we can generate various layouts that share the\nsame structure of the input example. In addition to this conditional generation\ndesign, we also integrate a differentiable rendering module that enables layout\nrefinement using only 2D projections of the scene. Given a depth and a\nsemantics map, the differentiable rendering module enables optimizing over the\nsynthesized layout to fit the given input in an analysis-by-synthesis fashion.\n"},{"type":"highlight","text":"Experiments suggest that our model achieves higher accuracy and diversity in\nconditional scene synthesis and allows exemplar-based scene generation from\nvarious input forms."}]},{"paper":{"id":"2407.14279","authors":[{"_id":"693be1309874a2a5e4ffb427","user":{"_id":"64d9f47a5a0d3c3c343ccda6","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64d9f47a5a0d3c3c343ccda6/itg8EDNuD52e66vMuhdu0.png","isPro":false,"fullname":"Rafay","user":"rafaymhddn","type":"user","name":"rafaymhddn"},"name":"Rafay Mohiuddin","status":"claimed_verified","statusLastChangedAt":"2025-12-12T12:59:31.392Z","hidden":false},{"_id":"693be1309874a2a5e4ffb428","name":"Sai Manoj Prakhya","hidden":false},{"_id":"693be1309874a2a5e4ffb429","name":"Fiona Collins","hidden":false},{"_id":"693be1309874a2a5e4ffb42a","name":"Ziyuan Liu","hidden":false},{"_id":"693be1309874a2a5e4ffb42b","name":"André Borrmann","hidden":false}],"publishedAt":"2024-07-19T13:01:12.000Z","title":"OpenSU3D: Open World 3D Scene Understanding using Foundation Models","summary":"In this paper, we present a novel, scalable approach for constructing open set, instance-level 3D scene representations, advancing open world understanding of 3D environments. Existing methods require pre-constructed 3D scenes and face scalability issues due to per-point feature vector learning, limiting their efficacy with complex queries. Our method overcomes these limitations by incrementally building instance-level 3D scene representations using 2D foundation models, efficiently aggregating instance-level details such as masks, feature vectors, names, and captions. We introduce fusion schemes for feature vectors to enhance their contextual knowledge and performance on complex queries. Additionally, we explore large language models for robust automatic annotation and spatial reasoning tasks. We evaluate our proposed approach on multiple scenes from ScanNet and Replica datasets demonstrating zero-shot generalization capabilities, exceeding current state-of-the-art methods in open world 3D scene understanding.","upvotes":0,"discussionId":"693be1309874a2a5e4ffb42c","projectPage":"https://opensu3d.github.io/","ai_summary":"A novel approach constructs scalable, open set 3D scene representations using 2D models and large language models, enabling zero-shot generalization and outperforming existing methods in open world understanding.","ai_keywords":["3D scene representations","open set","instance-level","2D foundation models","feature vector learning","feature vectors","fusion schemes","large language models","automatic annotation","spatial reasoning","ScanNet","Replica datasets","zero-shot generalization"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","organization":{"_id":"61fae781e68759322b9767be","name":"TUM","fullname":"Technical University of Munich","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/1661167219960-629521a0f937190946e15d7f.jpeg"}},"publishedAt":"2024-07-19T09:01:12.000Z","title":"OpenSU3D: Open World 3D Scene Understanding using Foundation Models","summary":"In this paper, we present a novel, scalable approach for constructing open set, instance-level 3D scene representations, advancing open world understanding of 3D environments. Existing methods require pre-constructed 3D scenes and face scalability issues due to per-point feature vector learning, limiting their efficacy with complex queries. Our method overcomes these limitations by incrementally building instance-level 3D scene representations using 2D foundation models, efficiently aggregating instance-level details such as masks, feature vectors, names, and captions. We introduce fusion schemes for feature vectors to enhance their contextual knowledge and performance on complex queries. Additionally, we explore large language models for robust automatic annotation and spatial reasoning tasks. We evaluate our proposed approach on multiple scenes from ScanNet and Replica datasets demonstrating zero-shot generalization capabilities, exceeding current state-of-the-art methods in open world 3D scene understanding.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2407.14279.png","numComments":0,"upvoted":false,"organization":{"_id":"61fae781e68759322b9767be","name":"TUM","fullname":"Technical University of Munich","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/1661167219960-629521a0f937190946e15d7f.jpeg"},"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"OpenSU3D: Open World 3D Scene Understanding using Foundation Models"}],"highlightedSummary":[{"type":"text","text":"In this paper, we present a novel, scalable approach for constructing open set, instance-level 3D scene representations, advancing open world understanding of 3D environments. Existing methods require pre-constructed 3D scenes and face scalability issues due to per-point feature vector learning, limiting their efficacy with complex queries. "},{"type":"highlight","text":"Our method overcomes these limitations by incrementally building instance-level 3D scene representations using 2D foundation models, efficiently aggregating instance-level details such as masks, feature vectors, names, and captions."},{"type":"text","text":" We introduce fusion schemes for feature vectors to enhance their contextual knowledge and performance on complex queries. Additionally, we explore large language models for robust automatic annotation and spatial reasoning tasks. We evaluate our proposed approach on multiple scenes from ScanNet and Replica datasets demonstrating zero-shot generalization capabilities, exceeding current state-of-the-art methods in open world 3D scene understanding."}]},{"paper":{"id":"2404.03421","authors":[{"_id":"660f9eea3b3e92e09d840fb6","user":{"_id":"6512b96497176e04b080df39","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6512b96497176e04b080df39/pbl6WwD5HGCQXNoUbP6-S.jpeg","isPro":false,"fullname":"Andreea Ardelean","user":"andreead-a","type":"user","name":"andreead-a"},"name":"Andreea Dogaru","status":"claimed_verified","statusLastChangedAt":"2024-04-05T08:40:00.793Z","hidden":false},{"_id":"660f9eea3b3e92e09d840fb7","name":"Mert Özer","hidden":false},{"_id":"660f9eea3b3e92e09d840fb8","name":"Bernhard Egger","hidden":false}],"publishedAt":"2024-04-04T12:58:46.000Z","title":"Generalizable 3D Scene Reconstruction via Divide and Conquer from a\n  Single View","summary":"Single-view 3D reconstruction is currently approached from two dominant\nperspectives: reconstruction of scenes with limited diversity using 3D data\nsupervision or reconstruction of diverse singular objects using large image\npriors. However, real-world scenarios are far more complex and exceed the\ncapabilities of these methods. We therefore propose a hybrid method following a\ndivide-and-conquer strategy. We first process the scene holistically,\nextracting depth and semantic information, and then leverage a single-shot\nobject-level method for the detailed reconstruction of individual components.\nBy following a compositional processing approach, the overall framework\nachieves full reconstruction of complex 3D scenes from a single image. We\npurposely design our pipeline to be highly modular by carefully integrating\nspecific procedures for each processing step, without requiring an end-to-end\ntraining of the whole system. This enables the pipeline to naturally improve as\nfuture methods can replace the individual modules. We demonstrate the\nreconstruction performance of our approach on both synthetic and real-world\nscenes, comparing favorable against prior works. Project page:\nhttps://andreeadogaru.github.io/Gen3DSR.","upvotes":4,"discussionId":"660f9eee3b3e92e09d841076","projectPage":"https://andreeadogaru.github.io/Gen3DSR/","githubRepo":"https://github.com/AndreeaDogaru/Gen3DSR","githubRepoAddedBy":"user","ai_summary":"A hybrid single-view 3D reconstruction method combines holistic scene processing with detailed object-level reconstruction, achieving full reconstruction of complex scenes using modular components.","ai_keywords":["single-view 3D reconstruction","depth estimation","semantic information","single-shot object-level reconstruction","compositional processing","modular pipeline"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":203},"publishedAt":"2024-04-04T08:58:46.000Z","title":"Generalizable 3D Scene Reconstruction via Divide and Conquer from a\n  Single View","summary":"Single-view 3D reconstruction is currently approached from two dominant\nperspectives: reconstruction of scenes with limited diversity using 3D data\nsupervision or reconstruction of diverse singular objects using large image\npriors. However, real-world scenarios are far more complex and exceed the\ncapabilities of these methods. We therefore propose a hybrid method following a\ndivide-and-conquer strategy. We first process the scene holistically,\nextracting depth and semantic information, and then leverage a single-shot\nobject-level method for the detailed reconstruction of individual components.\nBy following a compositional processing approach, the overall framework\nachieves full reconstruction of complex 3D scenes from a single image. We\npurposely design our pipeline to be highly modular by carefully integrating\nspecific procedures for each processing step, without requiring an end-to-end\ntraining of the whole system. This enables the pipeline to naturally improve as\nfuture methods can replace the individual modules. We demonstrate the\nreconstruction performance of our approach on both synthetic and real-world\nscenes, comparing favorable against prior works. Project page:\nhttps://andreeadogaru.github.io/Gen3DSR.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2404.03421.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Generalizable 3D Scene Reconstruction via Divide and Conquer from a\n  Single View"}],"highlightedSummary":[{"type":"text","text":"Single-view 3D reconstruction is currently approached from two dominant\nperspectives: reconstruction of scenes with limited diversity using 3D data\nsupervision or reconstruction of diverse singular objects using large image\npriors. However, real-world scenarios are far more complex and exceed the\ncapabilities of these methods. We therefore propose a hybrid method following a\ndivide-and-conquer strategy. We first process the scene holistically,\nextracting depth and semantic information, and then leverage a single-shot\nobject-level method for the detailed reconstruction of individual components.\n"},{"type":"highlight","text":"By following a compositional processing approach, the overall framework\nachieves full reconstruction of complex 3D scenes from a single image."},{"type":"text","text":" We\npurposely design our pipeline to be highly modular by carefully integrating\nspecific procedures for each processing step, without requiring an end-to-end\ntraining of the whole system. This enables the pipeline to naturally improve as\nfuture methods can replace the individual modules. We demonstrate the\nreconstruction performance of our approach on both synthetic and real-world\nscenes, comparing favorable against prior works. Project page:\nhttps://andreeadogaru.github.io/Gen3DSR."}]},{"paper":{"id":"2403.09439","authors":[{"_id":"689e9d1fa4caabb4320e5d30","name":"Frank Zhang","hidden":false},{"_id":"689e9d1fa4caabb4320e5d31","user":{"_id":"66bb4d475d27918b4bd8595f","avatarUrl":"/avatars/974ec48aa6d41166799300b2ff44a92c.svg","isPro":false,"fullname":"Yibo(SII)","user":"YiboZhang2001","type":"user","name":"YiboZhang2001"},"name":"Yibo Zhang","status":"claimed_verified","statusLastChangedAt":"2025-08-18T06:57:26.811Z","hidden":false},{"_id":"689e9d1fa4caabb4320e5d32","name":"Quan Zheng","hidden":false},{"_id":"689e9d1fa4caabb4320e5d33","name":"Rui Ma","hidden":false},{"_id":"689e9d1fa4caabb4320e5d34","name":"Wei Hua","hidden":false},{"_id":"689e9d1fa4caabb4320e5d35","name":"Hujun Bao","hidden":false},{"_id":"689e9d1fa4caabb4320e5d36","name":"Weiwei Xu","hidden":false},{"_id":"689e9d1fa4caabb4320e5d37","name":"Changqing Zou","hidden":false}],"publishedAt":"2024-03-14T14:31:22.000Z","title":"3D-SceneDreamer: Text-Driven 3D-Consistent Scene Generation","summary":"Text-driven 3D scene generation techniques have made rapid progress in recent\nyears. Their success is mainly attributed to using existing generative models\nto iteratively perform image warping and inpainting to generate 3D scenes.\nHowever, these methods heavily rely on the outputs of existing models, leading\nto error accumulation in geometry and appearance that prevent the models from\nbeing used in various scenarios (e.g., outdoor and unreal scenarios). To\naddress this limitation, we generatively refine the newly generated local views\nby querying and aggregating global 3D information, and then progressively\ngenerate the 3D scene. Specifically, we employ a tri-plane features-based NeRF\nas a unified representation of the 3D scene to constrain global 3D consistency,\nand propose a generative refinement network to synthesize new contents with\nhigher quality by exploiting the natural image prior from 2D diffusion model as\nwell as the global 3D information of the current scene. Our extensive\nexperiments demonstrate that, in comparison to previous methods, our approach\nsupports wide variety of scene generation and arbitrary camera trajectories\nwith improved visual quality and 3D consistency.","upvotes":0,"discussionId":"689e9d1fa4caabb4320e5d38","ai_summary":"A tri-plane features-based NeRF and generative refinement network improve 3D scene generation by leveraging global 3D information and 2D diffusion models, enhancing visual quality and consistency.","ai_keywords":["NeRF","tri-plane features","generative refinement network","2D diffusion model","3D consistency","scene generation","camera trajectories"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-03-14T10:31:22.000Z","title":"3D-SceneDreamer: Text-Driven 3D-Consistent Scene Generation","summary":"Text-driven 3D scene generation techniques have made rapid progress in recent\nyears. Their success is mainly attributed to using existing generative models\nto iteratively perform image warping and inpainting to generate 3D scenes.\nHowever, these methods heavily rely on the outputs of existing models, leading\nto error accumulation in geometry and appearance that prevent the models from\nbeing used in various scenarios (e.g., outdoor and unreal scenarios). To\naddress this limitation, we generatively refine the newly generated local views\nby querying and aggregating global 3D information, and then progressively\ngenerate the 3D scene. Specifically, we employ a tri-plane features-based NeRF\nas a unified representation of the 3D scene to constrain global 3D consistency,\nand propose a generative refinement network to synthesize new contents with\nhigher quality by exploiting the natural image prior from 2D diffusion model as\nwell as the global 3D information of the current scene. Our extensive\nexperiments demonstrate that, in comparison to previous methods, our approach\nsupports wide variety of scene generation and arbitrary camera trajectories\nwith improved visual quality and 3D consistency.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2403.09439.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"3D-SceneDreamer: Text-Driven 3D-Consistent Scene Generation"}],"highlightedSummary":[{"type":"text","text":"Text-driven 3D scene generation techniques have made rapid progress in recent\nyears. Their success is mainly attributed to using existing generative models\nto iteratively perform image warping and inpainting to generate 3D scenes.\n"},{"type":"highlight","text":"However, these methods heavily rely on the outputs of existing models, leading\nto error accumulation in geometry and appearance that prevent the models from\nbeing used in various scenarios (e.g., outdoor and unreal scenarios)."},{"type":"text","text":" To\naddress this limitation, we generatively refine the newly generated local views\nby querying and aggregating global 3D information, and then progressively\ngenerate the 3D scene. Specifically, we employ a tri-plane features-based NeRF\nas a unified representation of the 3D scene to constrain global 3D consistency,\nand propose a generative refinement network to synthesize new contents with\nhigher quality by exploiting the natural image prior from 2D diffusion model as\nwell as the global 3D information of the current scene. Our extensive\nexperiments demonstrate that, in comparison to previous methods, our approach\nsupports wide variety of scene generation and arbitrary camera trajectories\nwith improved visual quality and 3D consistency."}]},{"paper":{"id":"2505.15765","authors":[{"_id":"682e85aa7b41e70cf11758b6","name":"Kaizhi Zheng","hidden":false},{"_id":"682e85aa7b41e70cf11758b7","user":{"_id":"67f0985552f32a539aa00cb2","avatarUrl":"/avatars/4337511d15223342e2f2b150cbdcf2e7.svg","isPro":false,"fullname":"RUIJIAN ZHA","user":"rz2689","type":"user","name":"rz2689"},"name":"Ruijian Zhang","status":"claimed_verified","statusLastChangedAt":"2025-05-26T08:22:21.394Z","hidden":false},{"_id":"682e85aa7b41e70cf11758b8","name":"Jing Gu","hidden":false},{"_id":"682e85aa7b41e70cf11758b9","name":"Jie Yang","hidden":false},{"_id":"682e85aa7b41e70cf11758ba","user":{"_id":"64679a226192d39142245e5e","avatarUrl":"/avatars/05abee0b6317f100923936ca2099e9eb.svg","isPro":false,"fullname":"Xin Eric Wang","user":"xw-eric","type":"user","name":"xw-eric"},"name":"Xin Eric Wang","status":"claimed_verified","statusLastChangedAt":"2025-05-26T08:22:23.701Z","hidden":false}],"publishedAt":"2025-05-21T17:10:47.000Z","title":"Constructing a 3D Town from a Single Image","summary":"Acquiring detailed 3D scenes typically demands costly equipment, multi-view\ndata, or labor-intensive modeling. Therefore, a lightweight alternative,\ngenerating complex 3D scenes from a single top-down image, plays an essential\nrole in real-world applications. While recent 3D generative models have\nachieved remarkable results at the object level, their extension to full-scene\ngeneration often leads to inconsistent geometry, layout hallucinations, and\nlow-quality meshes. In this work, we introduce 3DTown, a training-free\nframework designed to synthesize realistic and coherent 3D scenes from a single\ntop-down view. Our method is grounded in two principles: region-based\ngeneration to improve image-to-3D alignment and resolution, and spatial-aware\n3D inpainting to ensure global scene coherence and high-quality geometry\ngeneration. Specifically, we decompose the input image into overlapping regions\nand generate each using a pretrained 3D object generator, followed by a masked\nrectified flow inpainting process that fills in missing geometry while\nmaintaining structural continuity. This modular design allows us to overcome\nresolution bottlenecks and preserve spatial structure without requiring 3D\nsupervision or fine-tuning. Extensive experiments across diverse scenes show\nthat 3DTown outperforms state-of-the-art baselines, including Trellis,\nHunyuan3D-2, and TripoSG, in terms of geometry quality, spatial coherence, and\ntexture fidelity. Our results demonstrate that high-quality 3D town generation\nis achievable from a single image using a principled, training-free approach.","upvotes":24,"discussionId":"682e85b17b41e70cf1175aa0","projectPage":"https://eric-ai-lab.github.io/3dtown.github.io/","ai_summary":"A training-free framework named 3DTown generates realistic 3D scenes from a single top-down image using region-based generation and spatial-aware 3D inpainting techniques.","ai_keywords":["3D generative models","3DTown","region-based generation","spatial-aware 3D inpainting","masked rectified flow","Trellis","Hunyuan3D-2","TripoSG"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-05-21T13:10:47.000Z","title":"Constructing a 3D Town from a Single Image","summary":"Acquiring detailed 3D scenes typically demands costly equipment, multi-view\ndata, or labor-intensive modeling. Therefore, a lightweight alternative,\ngenerating complex 3D scenes from a single top-down image, plays an essential\nrole in real-world applications. While recent 3D generative models have\nachieved remarkable results at the object level, their extension to full-scene\ngeneration often leads to inconsistent geometry, layout hallucinations, and\nlow-quality meshes. In this work, we introduce 3DTown, a training-free\nframework designed to synthesize realistic and coherent 3D scenes from a single\ntop-down view. Our method is grounded in two principles: region-based\ngeneration to improve image-to-3D alignment and resolution, and spatial-aware\n3D inpainting to ensure global scene coherence and high-quality geometry\ngeneration. Specifically, we decompose the input image into overlapping regions\nand generate each using a pretrained 3D object generator, followed by a masked\nrectified flow inpainting process that fills in missing geometry while\nmaintaining structural continuity. This modular design allows us to overcome\nresolution bottlenecks and preserve spatial structure without requiring 3D\nsupervision or fine-tuning. Extensive experiments across diverse scenes show\nthat 3DTown outperforms state-of-the-art baselines, including Trellis,\nHunyuan3D-2, and TripoSG, in terms of geometry quality, spatial coherence, and\ntexture fidelity. Our results demonstrate that high-quality 3D town generation\nis achievable from a single image using a principled, training-free approach.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2505.15765.png","numComments":3,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Constructing a 3D Town from a Single Image"}],"highlightedSummary":[{"type":"text","text":"Acquiring detailed 3D scenes typically demands costly equipment, multi-view\ndata, or labor-intensive modeling. Therefore, a lightweight alternative,\ngenerating complex 3D scenes from a single top-down image, plays an essential\nrole in real-world applications. "},{"type":"highlight","text":"While recent 3D generative models have\nachieved remarkable results at the object level, their extension to full-scene\ngeneration often leads to inconsistent geometry, layout hallucinations, and\nlow-quality meshes."},{"type":"text","text":" In this work, we introduce 3DTown, a training-free\nframework designed to synthesize realistic and coherent 3D scenes from a single\ntop-down view. Our method is grounded in two principles: region-based\ngeneration to improve image-to-3D alignment and resolution, and spatial-aware\n3D inpainting to ensure global scene coherence and high-quality geometry\ngeneration. Specifically, we decompose the input image into overlapping regions\nand generate each using a pretrained 3D object generator, followed by a masked\nrectified flow inpainting process that fills in missing geometry while\nmaintaining structural continuity. This modular design allows us to overcome\nresolution bottlenecks and preserve spatial structure without requiring 3D\nsupervision or fine-tuning. Extensive experiments across diverse scenes show\nthat 3DTown outperforms state-of-the-art baselines, including Trellis,\nHunyuan3D-2, and TripoSG, in terms of geometry quality, spatial coherence, and\ntexture fidelity. Our results demonstrate that high-quality 3D town generation\nis achievable from a single image using a principled, training-free approach."}]},{"paper":{"id":"2312.08885","authors":[{"_id":"65f0116a447e44e3d39232d1","user":{"_id":"63047acc41387c7f11756284","avatarUrl":"/avatars/a184fca1ee2d98b81e1917aecc3cdca7.svg","isPro":false,"fullname":"i3h","user":"qihang","type":"user","name":"qihang"},"name":"Qihang Zhang","status":"claimed_verified","statusLastChangedAt":"2024-10-14T07:49:20.370Z","hidden":false},{"_id":"65f0116a447e44e3d39232d2","name":"Chaoyang Wang","hidden":false},{"_id":"65f0116a447e44e3d39232d3","name":"Aliaksandr Siarohin","hidden":false},{"_id":"65f0116a447e44e3d39232d4","name":"Peiye Zhuang","hidden":false},{"_id":"65f0116a447e44e3d39232d5","user":{"_id":"6555da9adcf410fd0753569c","avatarUrl":"/avatars/ac58a796bb54f334fdb475a4b75c4d27.svg","isPro":false,"fullname":"Yinghao Xu","user":"justimyhxu","type":"user","name":"justimyhxu"},"name":"Yinghao Xu","status":"claimed_verified","statusLastChangedAt":"2026-02-04T12:39:41.401Z","hidden":false},{"_id":"65f0116a447e44e3d39232d6","name":"Ceyuan Yang","hidden":false},{"_id":"65f0116a447e44e3d39232d7","name":"Dahua Lin","hidden":false},{"_id":"65f0116a447e44e3d39232d8","name":"Bolei Zhou","hidden":false},{"_id":"65f0116a447e44e3d39232d9","name":"Sergey Tulyakov","hidden":false},{"_id":"65f0116a447e44e3d39232da","name":"Hsin-Ying Lee","hidden":false}],"publishedAt":"2023-12-13T18:59:30.000Z","title":"SceneWiz3D: Towards Text-guided 3D Scene Composition","summary":"We are witnessing significant breakthroughs in the technology for generating\n3D objects from text. Existing approaches either leverage large text-to-image\nmodels to optimize a 3D representation or train 3D generators on object-centric\ndatasets. Generating entire scenes, however, remains very challenging as a\nscene contains multiple 3D objects, diverse and scattered. In this work, we\nintroduce SceneWiz3D, a novel approach to synthesize high-fidelity 3D scenes\nfrom text. We marry the locality of objects with globality of scenes by\nintroducing a hybrid 3D representation: explicit for objects and implicit for\nscenes. Remarkably, an object, being represented explicitly, can be either\ngenerated from text using conventional text-to-3D approaches, or provided by\nusers. To configure the layout of the scene and automatically place objects, we\napply the Particle Swarm Optimization technique during the optimization\nprocess. Furthermore, it is difficult for certain parts of the scene (e.g.,\ncorners, occlusion) to receive multi-view supervision, leading to inferior\ngeometry. We incorporate an RGBD panorama diffusion model to mitigate it,\nresulting in high-quality geometry. Extensive evaluation supports that our\napproach achieves superior quality over previous approaches, enabling the\ngeneration of detailed and view-consistent 3D scenes.","upvotes":0,"discussionId":"65f0116c447e44e3d3923379","ai_summary":"A new method, SceneWiz3D, generates high-fidelity 3D scenes from text by combining explicit object representations with an implicit scene representation, optimizing layout with Particle Swarm Optimization and improving geometry with an RGBD panorama diffusion model.","ai_keywords":["3D generation","text-to-3D","hybrid 3D representation","explicit representation","implicit representation","Particle Swarm Optimization","RGBD panorama diffusion model","view-consistency"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2023-12-13T13:59:30.000Z","title":"SceneWiz3D: Towards Text-guided 3D Scene Composition","summary":"We are witnessing significant breakthroughs in the technology for generating\n3D objects from text. Existing approaches either leverage large text-to-image\nmodels to optimize a 3D representation or train 3D generators on object-centric\ndatasets. Generating entire scenes, however, remains very challenging as a\nscene contains multiple 3D objects, diverse and scattered. In this work, we\nintroduce SceneWiz3D, a novel approach to synthesize high-fidelity 3D scenes\nfrom text. We marry the locality of objects with globality of scenes by\nintroducing a hybrid 3D representation: explicit for objects and implicit for\nscenes. Remarkably, an object, being represented explicitly, can be either\ngenerated from text using conventional text-to-3D approaches, or provided by\nusers. To configure the layout of the scene and automatically place objects, we\napply the Particle Swarm Optimization technique during the optimization\nprocess. Furthermore, it is difficult for certain parts of the scene (e.g.,\ncorners, occlusion) to receive multi-view supervision, leading to inferior\ngeometry. We incorporate an RGBD panorama diffusion model to mitigate it,\nresulting in high-quality geometry. Extensive evaluation supports that our\napproach achieves superior quality over previous approaches, enabling the\ngeneration of detailed and view-consistent 3D scenes.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2312.08885.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"SceneWiz3D: Towards Text-guided 3D Scene Composition"}],"highlightedSummary":[{"type":"text","text":"We are witnessing significant breakthroughs in the technology for generating\n3D objects from text. Existing approaches either leverage large text-to-image\nmodels to optimize a 3D representation or train 3D generators on object-centric\ndatasets. Generating entire scenes, however, remains very challenging as a\nscene contains multiple 3D objects, diverse and scattered. In this work, we\nintroduce SceneWiz3D, a novel approach to synthesize high-fidelity 3D scenes\nfrom text. "},{"type":"highlight","text":"We marry the locality of objects with globality of scenes by\nintroducing a hybrid 3D representation: explicit for objects and implicit for\nscenes."},{"type":"text","text":" Remarkably, an object, being represented explicitly, can be either\ngenerated from text using conventional text-to-3D approaches, or provided by\nusers. To configure the layout of the scene and automatically place objects, we\napply the Particle Swarm Optimization technique during the optimization\nprocess. Furthermore, it is difficult for certain parts of the scene (e.g.,\ncorners, occlusion) to receive multi-view supervision, leading to inferior\ngeometry. We incorporate an RGBD panorama diffusion model to mitigate it,\nresulting in high-quality geometry. Extensive evaluation supports that our\napproach achieves superior quality over previous approaches, enabling the\ngeneration of detailed and view-consistent 3D scenes."}]},{"paper":{"id":"2402.07207","authors":[{"_id":"65cadb0805cc358d07f5dca4","name":"Xiaoyu Zhou","hidden":false},{"_id":"65cadb0805cc358d07f5dca5","user":{"_id":"691fe1f6ddc7986706bf86cf","avatarUrl":"/avatars/4f24fd572fa4a07d0d6cfcba8248e6d8.svg","isPro":false,"fullname":"Xingjian Ran","user":"xjran","type":"user","name":"xjran"},"name":"Xingjian Ran","status":"claimed_verified","statusLastChangedAt":"2026-09-04T08:45:03.904Z","hidden":false},{"_id":"65cadb0805cc358d07f5dca6","user":{"_id":"658976866cf9325ae6e3a10f","avatarUrl":"/avatars/85ef8bdc24191ee3ce3eb277d0223aad.svg","isPro":false,"fullname":"yajiao xiong","user":"yajiao","type":"user","name":"yajiao"},"name":"Yajiao Xiong","status":"admin_assigned","statusLastChangedAt":"2024-02-13T13:50:48.407Z","hidden":false},{"_id":"65cadb0805cc358d07f5dca7","name":"Jinlin He","hidden":false},{"_id":"65cadb0805cc358d07f5dca8","name":"Zhiwei Lin","hidden":false},{"_id":"65cadb0805cc358d07f5dca9","name":"Yongtao Wang","hidden":false},{"_id":"65cadb0805cc358d07f5dcaa","name":"Deqing Sun","hidden":false},{"_id":"65cadb0805cc358d07f5dcab","name":"Ming-Hsuan Yang","hidden":false}],"publishedAt":"2024-02-11T13:40:08.000Z","title":"GALA3D: Towards Text-to-3D Complex Scene Generation via Layout-guided\n  Generative Gaussian Splatting","summary":"We present GALA3D, generative 3D GAussians with LAyout-guided control, for\neffective compositional text-to-3D generation. We first utilize large language\nmodels (LLMs) to generate the initial layout and introduce a layout-guided 3D\nGaussian representation for 3D content generation with adaptive geometric\nconstraints. We then propose an object-scene compositional optimization\nmechanism with conditioned diffusion to collaboratively generate realistic 3D\nscenes with consistent geometry, texture, scale, and accurate interactions\namong multiple objects while simultaneously adjusting the coarse layout priors\nextracted from the LLMs to align with the generated scene. Experiments show\nthat GALA3D is a user-friendly, end-to-end framework for state-of-the-art\nscene-level 3D content generation and controllable editing while ensuring the\nhigh fidelity of object-level entities within the scene. Source codes and\nmodels will be available at https://gala3d.github.io/.","upvotes":10,"discussionId":"65cadb0b05cc358d07f5dd6e","ai_summary":"GALA3D is a generative framework for 3D content that uses large language models and conditioned diffusion for layout-guided, compositional scene generation, ensuring high fidelity and realistic object interactions.","ai_keywords":["large language models","LLMs","layout-guided","3D Gaussian","object-scene compositional optimization","conditioned diffusion",".scene-level 3D content generation","controllable editing","high fidelity","object-level entities"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-02-11T08:40:08.000Z","title":"GALA3D: Towards Text-to-3D Complex Scene Generation via Layout-guided\n  Generative Gaussian Splatting","summary":"We present GALA3D, generative 3D GAussians with LAyout-guided control, for\neffective compositional text-to-3D generation. We first utilize large language\nmodels (LLMs) to generate the initial layout and introduce a layout-guided 3D\nGaussian representation for 3D content generation with adaptive geometric\nconstraints. We then propose an object-scene compositional optimization\nmechanism with conditioned diffusion to collaboratively generate realistic 3D\nscenes with consistent geometry, texture, scale, and accurate interactions\namong multiple objects while simultaneously adjusting the coarse layout priors\nextracted from the LLMs to align with the generated scene. Experiments show\nthat GALA3D is a user-friendly, end-to-end framework for state-of-the-art\nscene-level 3D content generation and controllable editing while ensuring the\nhigh fidelity of object-level entities within the scene. Source codes and\nmodels will be available at https://gala3d.github.io/.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2402.07207.png","numComments":1,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"GALA3D: Towards Text-to-3D Complex Scene Generation via Layout-guided\n  Generative Gaussian Splatting"}],"highlightedSummary":[{"type":"text","text":"We present GALA3D, generative 3D GAussians with LAyout-guided control, for\neffective compositional text-to-3D generation. We first utilize large language\nmodels (LLMs) to generate the initial layout and introduce a layout-guided 3D\nGaussian representation for 3D content generation with adaptive geometric\nconstraints. "},{"type":"highlight","text":"We then propose an object-scene compositional optimization\nmechanism with conditioned diffusion to collaboratively generate realistic 3D\nscenes with consistent geometry, texture, scale, and accurate interactions\namong multiple objects while simultaneously adjusting the coarse layout priors\nextracted from the LLMs to align with the generated scene."},{"type":"text","text":" Experiments show\nthat GALA3D is a user-friendly, end-to-end framework for state-of-the-art\nscene-level 3D content generation and controllable editing while ensuring the\nhigh fidelity of object-level entities within the scene. Source codes and\nmodels will be available at https://gala3d.github.io/."}]},{"paper":{"id":"2402.03445","authors":[{"_id":"65f818dc26f86cf337a99ed5","name":"Titas Anciukevičius","hidden":false},{"_id":"65f818dc26f86cf337a99ed6","name":"Fabian Manhardt","hidden":false},{"_id":"65f818dc26f86cf337a99ed7","name":"Federico Tombari","hidden":false},{"_id":"65f818dc26f86cf337a99ed8","user":{"_id":"66749366b82e48404d157a66","avatarUrl":"/avatars/a56066447b0ebcc26c31ed2388a08707.svg","isPro":false,"fullname":"Paul Henderson","user":"pmh47","type":"user","name":"pmh47"},"name":"Paul Henderson","status":"claimed_verified","statusLastChangedAt":"2024-06-21T07:07:45.330Z","hidden":false}],"publishedAt":"2024-02-05T19:00:45.000Z","title":"Denoising Diffusion via Image-Based Rendering","summary":"Generating 3D scenes is a challenging open problem, which requires\nsynthesizing plausible content that is fully consistent in 3D space. While\nrecent methods such as neural radiance fields excel at view synthesis and 3D\nreconstruction, they cannot synthesize plausible details in unobserved regions\nsince they lack a generative capability. Conversely, existing generative\nmethods are typically not capable of reconstructing detailed, large-scale\nscenes in the wild, as they use limited-capacity 3D scene representations,\nrequire aligned camera poses, or rely on additional regularizers. In this work,\nwe introduce the first diffusion model able to perform fast, detailed\nreconstruction and generation of real-world 3D scenes. To achieve this, we make\nthree contributions. First, we introduce a new neural scene representation,\nIB-planes, that can efficiently and accurately represent large 3D scenes,\ndynamically allocating more capacity as needed to capture details visible in\neach image. Second, we propose a denoising-diffusion framework to learn a prior\nover this novel 3D scene representation, using only 2D images without the need\nfor any additional supervision signal such as masks or depths. This supports 3D\nreconstruction and generation in a unified architecture. Third, we develop a\nprincipled approach to avoid trivial 3D solutions when integrating the\nimage-based rendering with the diffusion model, by dropping out representations\nof some images. We evaluate the model on several challenging datasets of real\nand synthetic images, and demonstrate superior results on generation, novel\nview synthesis and 3D reconstruction.","upvotes":0,"discussionId":"65f818dd26f86cf337a9a052","ai_summary":"A new diffusion model with an IB-planes neural scene representation can efficiently reconstruct and generate detailed 3D scenes from 2D images without additional supervision.","ai_keywords":["neural radiance fields","diffusion model","view synthesis","3D reconstruction","3D scene representation","IB-planes","denoising-diffusion framework","image-based rendering","3D solutions","dropout"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-02-05T14:00:45.000Z","title":"Denoising Diffusion via Image-Based Rendering","summary":"Generating 3D scenes is a challenging open problem, which requires\nsynthesizing plausible content that is fully consistent in 3D space. While\nrecent methods such as neural radiance fields excel at view synthesis and 3D\nreconstruction, they cannot synthesize plausible details in unobserved regions\nsince they lack a generative capability. Conversely, existing generative\nmethods are typically not capable of reconstructing detailed, large-scale\nscenes in the wild, as they use limited-capacity 3D scene representations,\nrequire aligned camera poses, or rely on additional regularizers. In this work,\nwe introduce the first diffusion model able to perform fast, detailed\nreconstruction and generation of real-world 3D scenes. To achieve this, we make\nthree contributions. First, we introduce a new neural scene representation,\nIB-planes, that can efficiently and accurately represent large 3D scenes,\ndynamically allocating more capacity as needed to capture details visible in\neach image. Second, we propose a denoising-diffusion framework to learn a prior\nover this novel 3D scene representation, using only 2D images without the need\nfor any additional supervision signal such as masks or depths. This supports 3D\nreconstruction and generation in a unified architecture. Third, we develop a\nprincipled approach to avoid trivial 3D solutions when integrating the\nimage-based rendering with the diffusion model, by dropping out representations\nof some images. We evaluate the model on several challenging datasets of real\nand synthetic images, and demonstrate superior results on generation, novel\nview synthesis and 3D reconstruction.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2402.03445.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Denoising Diffusion via Image-Based Rendering"}],"highlightedSummary":[{"type":"highlight","text":"Generating 3D scenes is a challenging open problem, which requires\nsynthesizing plausible content that is fully consistent in 3D space."},{"type":"text","text":" While\nrecent methods such as neural radiance fields excel at view synthesis and 3D\nreconstruction, they cannot synthesize plausible details in unobserved regions\nsince they lack a generative capability. Conversely, existing generative\nmethods are typically not capable of reconstructing detailed, large-scale\nscenes in the wild, as they use limited-capacity 3D scene representations,\nrequire aligned camera poses, or rely on additional regularizers. In this work,\nwe introduce the first diffusion model able to perform fast, detailed\nreconstruction and generation of real-world 3D scenes. To achieve this, we make\nthree contributions. First, we introduce a new neural scene representation,\nIB-planes, that can efficiently and accurately represent large 3D scenes,\ndynamically allocating more capacity as needed to capture details visible in\neach image. Second, we propose a denoising-diffusion framework to learn a prior\nover this novel 3D scene representation, using only 2D images without the need\nfor any additional supervision signal such as masks or depths. This supports 3D\nreconstruction and generation in a unified architecture. Third, we develop a\nprincipled approach to avoid trivial 3D solutions when integrating the\nimage-based rendering with the diffusion model, by dropping out representations\nof some images. We evaluate the model on several challenging datasets of real\nand synthetic images, and demonstrate superior results on generation, novel\nview synthesis and 3D reconstruction."}]},{"paper":{"id":"2601.15664","authors":[{"_id":"69731856fb12c92b735b7568","name":"Hongyang Wei","hidden":false},{"_id":"69731856fb12c92b735b7569","user":{"_id":"636e19078ba65db4a093a3f4","avatarUrl":"/avatars/287b063b44a022d8576256e80e489c31.svg","isPro":false,"fullname":"alexiosss","user":"Alexislhb","type":"user","name":"Alexislhb"},"name":"Hongbo Liu","status":"claimed_verified","statusLastChangedAt":"2026-08-28T13:25:36.982Z","hidden":false},{"_id":"69731856fb12c92b735b756a","name":"Zidong Wang","hidden":false},{"_id":"69731856fb12c92b735b756b","name":"Yi Peng","hidden":false},{"_id":"69731856fb12c92b735b756c","name":"Baixin Xu","hidden":false},{"_id":"69731856fb12c92b735b756d","name":"Size Wu","hidden":false},{"_id":"69731856fb12c92b735b756e","name":"Xuying Zhang","hidden":false},{"_id":"69731856fb12c92b735b756f","name":"Xianglong He","hidden":false},{"_id":"69731856fb12c92b735b7570","name":"Zexiang Liu","hidden":false},{"_id":"69731856fb12c92b735b7571","name":"Peiyu Wang","hidden":false},{"_id":"69731856fb12c92b735b7572","name":"Xuchen Song","hidden":false},{"_id":"69731856fb12c92b735b7573","name":"Yangguang Li","hidden":false},{"_id":"69731856fb12c92b735b7574","name":"Yang Liu","hidden":false},{"_id":"69731856fb12c92b735b7575","name":"Yahui Zhou","hidden":false}],"publishedAt":"2026-01-22T05:23:20.000Z","title":"Skywork UniPic 3.0: Unified Multi-Image Composition via Sequence Modeling","summary":"The recent surge in popularity of Nano-Banana and Seedream 4.0 underscores the community's strong interest in multi-image composition tasks. Compared to single-image editing, multi-image composition presents significantly greater challenges in terms of consistency and quality, yet existing models have not disclosed specific methodological details for achieving high-quality fusion. Through statistical analysis, we identify Human-Object Interaction (HOI) as the most sought-after category by the community. We therefore systematically analyze and implement a state-of-the-art solution for multi-image composition with a primary focus on HOI-centric tasks. We present Skywork UniPic 3.0, a unified multimodal framework that integrates single-image editing and multi-image composition. Our model supports an arbitrary (1~6) number and resolution of input images, as well as arbitrary output resolutions (within a total pixel budget of 1024x1024). To address the challenges of multi-image composition, we design a comprehensive data collection, filtering, and synthesis pipeline, achieving strong performance with only 700K high-quality training samples. Furthermore, we introduce a novel training paradigm that formulates multi-image composition as a sequence-modeling problem, transforming conditional generation into unified sequence synthesis. To accelerate inference, we integrate trajectory mapping and distribution matching into the post-training stage, enabling the model to produce high-fidelity samples in just 8 steps and achieve a 12.5x speedup over standard synthesis sampling. Skywork UniPic 3.0 achieves state-of-the-art performance on single-image editing benchmark and surpasses both Nano-Banana and Seedream 4.0 on multi-image composition benchmark, thereby validating the effectiveness of our data pipeline and training paradigm. Code, models and dataset are publicly available.","upvotes":4,"discussionId":"69731857fb12c92b735b7576","ai_summary":"Skywork UniPic 3.0 is a unified multimodal framework that advances multi-image composition through novel data pipelines, training paradigms, and efficient inference techniques.","ai_keywords":["multi-image composition","single-image editing","Human-Object Interaction","sequence-modeling problem","conditional generation","unified sequence synthesis","trajectory mapping","distribution matching","post-training stage","high-fidelity samples"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2026-01-22T00:23:20.000Z","title":"Skywork UniPic 3.0: Unified Multi-Image Composition via Sequence Modeling","summary":"The recent surge in popularity of Nano-Banana and Seedream 4.0 underscores the community's strong interest in multi-image composition tasks. Compared to single-image editing, multi-image composition presents significantly greater challenges in terms of consistency and quality, yet existing models have not disclosed specific methodological details for achieving high-quality fusion. Through statistical analysis, we identify Human-Object Interaction (HOI) as the most sought-after category by the community. We therefore systematically analyze and implement a state-of-the-art solution for multi-image composition with a primary focus on HOI-centric tasks. We present Skywork UniPic 3.0, a unified multimodal framework that integrates single-image editing and multi-image composition. Our model supports an arbitrary (1~6) number and resolution of input images, as well as arbitrary output resolutions (within a total pixel budget of 1024x1024). To address the challenges of multi-image composition, we design a comprehensive data collection, filtering, and synthesis pipeline, achieving strong performance with only 700K high-quality training samples. Furthermore, we introduce a novel training paradigm that formulates multi-image composition as a sequence-modeling problem, transforming conditional generation into unified sequence synthesis. To accelerate inference, we integrate trajectory mapping and distribution matching into the post-training stage, enabling the model to produce high-fidelity samples in just 8 steps and achieve a 12.5x speedup over standard synthesis sampling. Skywork UniPic 3.0 achieves state-of-the-art performance on single-image editing benchmark and surpasses both Nano-Banana and Seedream 4.0 on multi-image composition benchmark, thereby validating the effectiveness of our data pipeline and training paradigm. Code, models and dataset are publicly available.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2601.15664.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Skywork UniPic 3.0: Unified Multi-Image Composition via Sequence Modeling"}],"highlightedSummary":[{"type":"text","text":"The recent surge in popularity of Nano-Banana and Seedream 4.0 underscores the community's strong interest in multi-image composition tasks. Compared to single-image editing, multi-image composition presents significantly greater challenges in terms of consistency and quality, yet existing models have not disclosed specific methodological details for achieving high-quality fusion. Through statistical analysis, we identify Human-Object Interaction (HOI) as the most sought-after category by the community. We therefore systematically analyze and implement a state-of-the-art solution for multi-image composition with a primary focus on HOI-centric tasks. We present Skywork UniPic 3.0, a unified multimodal framework that integrates single-image editing and multi-image composition. Our model supports an arbitrary (1~6) number and resolution of input images, as well as arbitrary output resolutions (within a total pixel budget of 1024x1024). To address the challenges of multi-image composition, we design a comprehensive data collection, filtering, and synthesis pipeline, achieving strong performance with only 700K high-quality training samples. "},{"type":"highlight","text":"Furthermore, we introduce a novel training paradigm that formulates multi-image composition as a sequence-modeling problem, transforming conditional generation into unified sequence synthesis."},{"type":"text","text":" To accelerate inference, we integrate trajectory mapping and distribution matching into the post-training stage, enabling the model to produce high-fidelity samples in just 8 steps and achieve a 12.5x speedup over standard synthesis sampling. Skywork UniPic 3.0 achieves state-of-the-art performance on single-image editing benchmark and surpasses both Nano-Banana and Seedream 4.0 on multi-image composition benchmark, thereby validating the effectiveness of our data pipeline and training paradigm. Code, models and dataset are publicly available."}]},{"paper":{"id":"2502.13951","authors":[{"_id":"67e92f206221f58722417f03","user":{"_id":"668569d06be8cd35515b9be2","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/668569d06be8cd35515b9be2/50Nggue0FIZfCrQi_8x7P.jpeg","isPro":false,"fullname":"Sara Dorfman","user":"saradorfman","type":"user","name":"saradorfman"},"name":"Sara Dorfman","status":"claimed_verified","statusLastChangedAt":"2026-06-25T10:11:49.774Z","hidden":false},{"_id":"67e92f206221f58722417f04","name":"Dana Cohen-Bar","hidden":false},{"_id":"67e92f206221f58722417f05","name":"Rinon Gal","hidden":false},{"_id":"67e92f206221f58722417f06","name":"Daniel Cohen-Or","hidden":false}],"publishedAt":"2025-02-19T18:49:31.000Z","title":"IP-Composer: Semantic Composition of Visual Concepts","summary":"Content creators often draw inspiration from multiple visual sources,\ncombining distinct elements to craft new compositions. Modern computational\napproaches now aim to emulate this fundamental creative process. Although\nrecent diffusion models excel at text-guided compositional synthesis, text as a\nmedium often lacks precise control over visual details. Image-based composition\napproaches can capture more nuanced features, but existing methods are\ntypically limited in the range of concepts they can capture, and require\nexpensive training procedures or specialized data. We present IP-Composer, a\nnovel training-free approach for compositional image generation that leverages\nmultiple image references simultaneously, while using natural language to\ndescribe the concept to be extracted from each image. Our method builds on\nIP-Adapter, which synthesizes novel images conditioned on an input image's CLIP\nembedding. We extend this approach to multiple visual inputs by crafting\ncomposite embeddings, stitched from the projections of multiple input images\nonto concept-specific CLIP-subspaces identified through text. Through\ncomprehensive evaluation, we show that our approach enables more precise\ncontrol over a larger range of visual concept compositions.","upvotes":2,"discussionId":"67e92f236221f58722417f75","ai_summary":"IP-Composer uses multiple image references and text descriptions to generate precise and diverse compositional images.","ai_keywords":["diffusion models","image-based composition","CLIP embedding","IP-Adapter","CLIP-subspaces","compositional image generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-02-19T13:49:31.000Z","title":"IP-Composer: Semantic Composition of Visual Concepts","summary":"Content creators often draw inspiration from multiple visual sources,\ncombining distinct elements to craft new compositions. Modern computational\napproaches now aim to emulate this fundamental creative process. Although\nrecent diffusion models excel at text-guided compositional synthesis, text as a\nmedium often lacks precise control over visual details. Image-based composition\napproaches can capture more nuanced features, but existing methods are\ntypically limited in the range of concepts they can capture, and require\nexpensive training procedures or specialized data. We present IP-Composer, a\nnovel training-free approach for compositional image generation that leverages\nmultiple image references simultaneously, while using natural language to\ndescribe the concept to be extracted from each image. Our method builds on\nIP-Adapter, which synthesizes novel images conditioned on an input image's CLIP\nembedding. We extend this approach to multiple visual inputs by crafting\ncomposite embeddings, stitched from the projections of multiple input images\nonto concept-specific CLIP-subspaces identified through text. Through\ncomprehensive evaluation, we show that our approach enables more precise\ncontrol over a larger range of visual concept compositions.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2502.13951.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"IP-Composer: Semantic Composition of Visual Concepts"}],"highlightedSummary":[{"type":"text","text":"Content creators often draw inspiration from multiple visual sources,\ncombining distinct elements to craft new compositions. Modern computational\napproaches now aim to emulate this fundamental creative process. Although\nrecent diffusion models excel at text-guided compositional synthesis, text as a\nmedium often lacks precise control over visual details. Image-based composition\napproaches can capture more nuanced features, but existing methods are\ntypically limited in the range of concepts they can capture, and require\nexpensive training procedures or specialized data. "},{"type":"highlight","text":"We present IP-Composer, a\nnovel training-free approach for compositional image generation that leverages\nmultiple image references simultaneously, while using natural language to\ndescribe the concept to be extracted from each image."},{"type":"text","text":" Our method builds on\nIP-Adapter, which synthesizes novel images conditioned on an input image's CLIP\nembedding. We extend this approach to multiple visual inputs by crafting\ncomposite embeddings, stitched from the projections of multiple input images\nonto concept-specific CLIP-subspaces identified through text. Through\ncomprehensive evaluation, we show that our approach enables more precise\ncontrol over a larger range of visual concept compositions."}]},{"paper":{"id":"2501.01424","authors":[{"_id":"67a42f2039b2bd4f5538967c","name":"Gaurav Parmar","hidden":false},{"_id":"67a42f2039b2bd4f5538967d","user":{"_id":"62853516e483e0d37b354ce1","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/62853516e483e0d37b354ce1/t5Tyd3E07w26B9Z3XpZWI.jpeg","isPro":false,"fullname":"Or Patashnik","user":"orpatashnik","type":"user","name":"orpatashnik"},"name":"Or Patashnik","status":"claimed_verified","statusLastChangedAt":"2025-10-16T06:44:06.930Z","hidden":false},{"_id":"67a42f2039b2bd4f5538967e","name":"Kuan-Chieh Wang","hidden":false},{"_id":"67a42f2039b2bd4f5538967f","name":"Daniil Ostashev","hidden":false},{"_id":"67a42f2039b2bd4f55389680","name":"Srinivasa Narasimhan","hidden":false},{"_id":"67a42f2039b2bd4f55389681","name":"Jun-Yan Zhu","hidden":false},{"_id":"67a42f2039b2bd4f55389682","name":"Daniel Cohen-Or","hidden":false},{"_id":"67a42f2039b2bd4f55389683","name":"Kfir Aberman","hidden":false}],"publishedAt":"2025-01-02T18:59:44.000Z","title":"Object-level Visual Prompts for Compositional Image Generation","summary":"We introduce a method for composing object-level visual prompts within a\ntext-to-image diffusion model. Our approach addresses the task of generating\nsemantically coherent compositions across diverse scenes and styles, similar to\nthe versatility and expressiveness offered by text prompts. A key challenge in\nthis task is to preserve the identity of the objects depicted in the input\nvisual prompts, while also generating diverse compositions across different\nimages. To address this challenge, we introduce a new KV-mixed cross-attention\nmechanism, in which keys and values are learned from distinct visual\nrepresentations. The keys are derived from an encoder with a small bottleneck\nfor layout control, whereas the values come from a larger bottleneck encoder\nthat captures fine-grained appearance details. By mixing keys and values from\nthese complementary sources, our model preserves the identity of the visual\nprompts while supporting flexible variations in object arrangement, pose, and\ncomposition. During inference, we further propose object-level compositional\nguidance to improve the method's identity preservation and layout correctness.\nResults show that our technique produces diverse scene compositions that\npreserve the unique characteristics of each visual prompt, expanding the\ncreative potential of text-to-image generation.","upvotes":1,"discussionId":"67a42f2539b2bd4f5538976b","ai_summary":"A KV-mixed cross-attention mechanism within a text-to-image diffusion model enhances object identity preservation while enabling diverse scene compositions.","ai_keywords":["text-to-image diffusion model","KV-mixed cross-attention","keys","values","encoder","layout control","fine-grained appearance details","object-level compositional guidance","identity preservation","layout correctness"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-01-02T13:59:44.000Z","title":"Object-level Visual Prompts for Compositional Image Generation","summary":"We introduce a method for composing object-level visual prompts within a\ntext-to-image diffusion model. Our approach addresses the task of generating\nsemantically coherent compositions across diverse scenes and styles, similar to\nthe versatility and expressiveness offered by text prompts. A key challenge in\nthis task is to preserve the identity of the objects depicted in the input\nvisual prompts, while also generating diverse compositions across different\nimages. To address this challenge, we introduce a new KV-mixed cross-attention\nmechanism, in which keys and values are learned from distinct visual\nrepresentations. The keys are derived from an encoder with a small bottleneck\nfor layout control, whereas the values come from a larger bottleneck encoder\nthat captures fine-grained appearance details. By mixing keys and values from\nthese complementary sources, our model preserves the identity of the visual\nprompts while supporting flexible variations in object arrangement, pose, and\ncomposition. During inference, we further propose object-level compositional\nguidance to improve the method's identity preservation and layout correctness.\nResults show that our technique produces diverse scene compositions that\npreserve the unique characteristics of each visual prompt, expanding the\ncreative potential of text-to-image generation.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2501.01424.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Object-level Visual Prompts for Compositional Image Generation"}],"highlightedSummary":[{"type":"text","text":"We introduce a method for composing object-level visual prompts within a\ntext-to-image diffusion model. Our approach addresses the task of generating\nsemantically coherent compositions across diverse scenes and styles, similar to\nthe versatility and expressiveness offered by text prompts. A key challenge in\nthis task is to preserve the identity of the objects depicted in the input\nvisual prompts, while also generating diverse compositions across different\nimages. To address this challenge, we introduce a new KV-mixed cross-attention\nmechanism, in which keys and values are learned from distinct visual\nrepresentations. The keys are derived from an encoder with a small bottleneck\nfor layout control, whereas the values come from a larger bottleneck encoder\nthat captures fine-grained appearance details. By mixing keys and values from\nthese complementary sources, our model preserves the identity of the visual\nprompts while supporting flexible variations in object arrangement, pose, and\ncomposition. During inference, we further propose object-level compositional\nguidance to improve the method's identity preservation and layout correctness.\n"},{"type":"highlight","text":"Results show that our technique produces diverse scene compositions that\npreserve the unique characteristics of each visual prompt, expanding the\ncreative potential of text-to-image generation."}]},{"paper":{"id":"2411.10913","authors":[{"_id":"673fc0c4bf2e9c35b2f51502","user":{"_id":"673f2840be306dfff3d306df","avatarUrl":"/avatars/6d8f9fcaf1413e699afd82d62d4c5291.svg","isPro":false,"fullname":"Alessandro Fontanella","user":"a-fontanella","type":"user","name":"a-fontanella"},"name":"Alessandro Fontanella","status":"claimed_verified","statusLastChangedAt":"2024-11-22T10:02:04.397Z","hidden":false},{"_id":"673fc0c4bf2e9c35b2f51503","user":{"_id":"63d44162ff1384ce6c636dce","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1674854731168-noauth.jpeg","isPro":false,"fullname":"Petru-Daniel Tudosiu","user":"danieltudosiu","type":"user","name":"danieltudosiu"},"name":"Petru-Daniel Tudosiu","status":"admin_assigned","statusLastChangedAt":"2024-11-22T13:22:23.915Z","hidden":false},{"_id":"673fc0c4bf2e9c35b2f51504","user":{"_id":"62b6bdaadd998a8b1e3df69b","avatarUrl":"/avatars/c845b5fdc487430eedf2519675a2b455.svg","isPro":false,"fullname":"Yongxin Yang","user":"HalleyX","type":"user","name":"HalleyX"},"name":"Yongxin Yang","status":"admin_assigned","statusLastChangedAt":"2024-11-22T13:22:30.375Z","hidden":false},{"_id":"673fc0c4bf2e9c35b2f51505","name":"Shifeng Zhang","hidden":false},{"_id":"673fc0c4bf2e9c35b2f51506","name":"Sarah Parisot","hidden":false}],"publishedAt":"2024-11-16T23:44:14.000Z","title":"Generating Compositional Scenes via Text-to-image RGBA Instance\n  Generation","summary":"Text-to-image diffusion generative models can generate high quality images at\nthe cost of tedious prompt engineering. Controllability can be improved by\nintroducing layout conditioning, however existing methods lack layout editing\nability and fine-grained control over object attributes. The concept of\nmulti-layer generation holds great potential to address these limitations,\nhowever generating image instances concurrently to scene composition limits\ncontrol over fine-grained object attributes, relative positioning in 3D space\nand scene manipulation abilities. In this work, we propose a novel multi-stage\ngeneration paradigm that is designed for fine-grained control, flexibility and\ninteractivity. To ensure control over instance attributes, we devise a novel\ntraining paradigm to adapt a diffusion model to generate isolated scene\ncomponents as RGBA images with transparency information. To build complex\nimages, we employ these pre-generated instances and introduce a multi-layer\ncomposite generation process that smoothly assembles components in realistic\nscenes. Our experiments show that our RGBA diffusion model is capable of\ngenerating diverse and high quality instances with precise control over object\nattributes. Through multi-layer composition, we demonstrate that our approach\nallows to build and manipulate images from highly complex prompts with\nfine-grained control over object appearance and location, granting a higher\ndegree of control than competing methods.","upvotes":4,"discussionId":"673fc0ccbf2e9c35b2f516a0","projectPage":"https://mulanrgba.github.io/","ai_summary":"A multi-stage diffusion model generates high-quality images with fine-grained control over object attributes and scene composition through a transparent RGBA diffusion model and multi-layer composition.","ai_keywords":["diffusion generative models","prompt engineering","layout conditioning","multi-layer generation","multi-stage generation","RGBA images","composite generation","scene components","object attributes","scene manipulation","precise control","object appearance","location"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-11-16T18:44:14.000Z","title":"Generating Compositional Scenes via Text-to-image RGBA Instance\n  Generation","summary":"Text-to-image diffusion generative models can generate high quality images at\nthe cost of tedious prompt engineering. Controllability can be improved by\nintroducing layout conditioning, however existing methods lack layout editing\nability and fine-grained control over object attributes. The concept of\nmulti-layer generation holds great potential to address these limitations,\nhowever generating image instances concurrently to scene composition limits\ncontrol over fine-grained object attributes, relative positioning in 3D space\nand scene manipulation abilities. In this work, we propose a novel multi-stage\ngeneration paradigm that is designed for fine-grained control, flexibility and\ninteractivity. To ensure control over instance attributes, we devise a novel\ntraining paradigm to adapt a diffusion model to generate isolated scene\ncomponents as RGBA images with transparency information. To build complex\nimages, we employ these pre-generated instances and introduce a multi-layer\ncomposite generation process that smoothly assembles components in realistic\nscenes. Our experiments show that our RGBA diffusion model is capable of\ngenerating diverse and high quality instances with precise control over object\nattributes. Through multi-layer composition, we demonstrate that our approach\nallows to build and manipulate images from highly complex prompts with\nfine-grained control over object appearance and location, granting a higher\ndegree of control than competing methods.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2411.10913.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Generating Compositional Scenes via Text-to-image RGBA Instance\n  Generation"}],"highlightedSummary":[{"type":"text","text":"Text-to-image diffusion generative models can generate high quality images at\nthe cost of tedious prompt engineering. Controllability can be improved by\nintroducing layout conditioning, however existing methods lack layout editing\nability and fine-grained control over object attributes. The concept of\nmulti-layer generation holds great potential to address these limitations,\nhowever generating image instances concurrently to scene composition limits\ncontrol over fine-grained object attributes, relative positioning in 3D space\nand scene manipulation abilities. In this work, we propose a novel multi-stage\ngeneration paradigm that is designed for fine-grained control, flexibility and\ninteractivity. To ensure control over instance attributes, we devise a novel\ntraining paradigm to adapt a diffusion model to generate isolated scene\ncomponents as RGBA images with transparency information. "},{"type":"highlight","text":"To build complex\nimages, we employ these pre-generated instances and introduce a multi-layer\ncomposite generation process that smoothly assembles components in realistic\nscenes."},{"type":"text","text":" Our experiments show that our RGBA diffusion model is capable of\ngenerating diverse and high quality instances with precise control over object\nattributes. Through multi-layer composition, we demonstrate that our approach\nallows to build and manipulate images from highly complex prompts with\nfine-grained control over object appearance and location, granting a higher\ndegree of control than competing methods."}]},{"paper":{"id":"2504.13072","authors":[{"_id":"6805bfc5e332a61dd90160b0","name":"Wenqi Dong","hidden":false},{"_id":"6805bfc5e332a61dd90160b1","user":{"_id":"63d748ff6f49aa82306b7e48","avatarUrl":"/avatars/9f0b8b8a09b14d76e52ed1bd312e6b63.svg","isPro":false,"fullname":"BB Yang","user":"ybbbbt","type":"user","name":"ybbbbt"},"name":"Bangbang Yang","status":"claimed_verified","statusLastChangedAt":"2025-04-22T09:52:51.612Z","hidden":false},{"_id":"6805bfc5e332a61dd90160b2","user":{"_id":"666830ef15929be31e37b0f1","avatarUrl":"/avatars/e737b9d0bed8ddd4d4837e90cbc09bbb.svg","isPro":false,"fullname":"Zesong Yang","user":"YZsZY","type":"user","name":"YZsZY"},"name":"Zesong Yang","status":"claimed_verified","statusLastChangedAt":"2026-09-29T08:45:05.207Z","hidden":false},{"_id":"6805bfc5e332a61dd90160b3","name":"Yuan Li","hidden":false},{"_id":"6805bfc5e332a61dd90160b4","user":{"_id":"638f3c7fb0525fa370494de8","avatarUrl":"/avatars/39f51d218af0008f2000a7bd221841fe.svg","isPro":false,"fullname":"Tao Hu","user":"yihouxiang","type":"user","name":"yihouxiang"},"name":"Tao Hu","status":"claimed_verified","statusLastChangedAt":"2025-07-02T07:41:39.393Z","hidden":false},{"_id":"6805bfc5e332a61dd90160b5","name":"Hujun Bao","hidden":false},{"_id":"6805bfc5e332a61dd90160b6","name":"Yuewen Ma","hidden":false},{"_id":"6805bfc5e332a61dd90160b7","name":"Zhaopeng Cui","hidden":false}],"publishedAt":"2025-04-17T16:33:39.000Z","title":"HiScene: Creating Hierarchical 3D Scenes with Isometric View Generation","summary":"Scene-level 3D generation represents a critical frontier in multimedia and\ncomputer graphics, yet existing approaches either suffer from limited object\ncategories or lack editing flexibility for interactive applications. In this\npaper, we present HiScene, a novel hierarchical framework that bridges the gap\nbetween 2D image generation and 3D object generation and delivers high-fidelity\nscenes with compositional identities and aesthetic scene content. Our key\ninsight is treating scenes as hierarchical \"objects\" under isometric views,\nwhere a room functions as a complex object that can be further decomposed into\nmanipulatable items. This hierarchical approach enables us to generate 3D\ncontent that aligns with 2D representations while maintaining compositional\nstructure. To ensure completeness and spatial alignment of each decomposed\ninstance, we develop a video-diffusion-based amodal completion technique that\neffectively handles occlusions and shadows between objects, and introduce shape\nprior injection to ensure spatial coherence within the scene. Experimental\nresults demonstrate that our method produces more natural object arrangements\nand complete object instances suitable for interactive applications, while\nmaintaining physical plausibility and alignment with user inputs.","upvotes":13,"discussionId":"6805bfc9e332a61dd901618b","projectPage":"https://zju3dv.github.io/hiscene/","ai_summary":"A hierarchical framework called HiScene combines 2D image and 3D object generation using video-diffusion for amodal completion and shape priors to produce coherent, high-fidelity, interactive 3D scenes.","ai_keywords":["hierarchical framework","2D image generation","3D object generation","isometric views","video-diffusion","amodal completion","spatial coherence","compositional identities","aesthetic scene content","natural object arrangements","physical plausibility","user inputs"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-04-17T12:33:39.000Z","title":"HiScene: Creating Hierarchical 3D Scenes with Isometric View Generation","summary":"Scene-level 3D generation represents a critical frontier in multimedia and\ncomputer graphics, yet existing approaches either suffer from limited object\ncategories or lack editing flexibility for interactive applications. In this\npaper, we present HiScene, a novel hierarchical framework that bridges the gap\nbetween 2D image generation and 3D object generation and delivers high-fidelity\nscenes with compositional identities and aesthetic scene content. Our key\ninsight is treating scenes as hierarchical \"objects\" under isometric views,\nwhere a room functions as a complex object that can be further decomposed into\nmanipulatable items. This hierarchical approach enables us to generate 3D\ncontent that aligns with 2D representations while maintaining compositional\nstructure. To ensure completeness and spatial alignment of each decomposed\ninstance, we develop a video-diffusion-based amodal completion technique that\neffectively handles occlusions and shadows between objects, and introduce shape\nprior injection to ensure spatial coherence within the scene. Experimental\nresults demonstrate that our method produces more natural object arrangements\nand complete object instances suitable for interactive applications, while\nmaintaining physical plausibility and alignment with user inputs.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2504.13072.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"HiScene: Creating Hierarchical 3D Scenes with Isometric View Generation"}],"highlightedSummary":[{"type":"text","text":"Scene-level 3D generation represents a critical frontier in multimedia and\ncomputer graphics, yet existing approaches either suffer from limited object\ncategories or lack editing flexibility for interactive applications. In this\npaper, we present HiScene, a novel hierarchical framework that bridges the gap\nbetween 2D image generation and 3D object generation and delivers high-fidelity\nscenes with compositional identities and aesthetic scene content. Our key\ninsight is treating scenes as hierarchical \"objects\" under isometric views,\nwhere a room functions as a complex object that can be further decomposed into\nmanipulatable items. This hierarchical approach enables us to generate 3D\ncontent that aligns with 2D representations while maintaining compositional\nstructure. To ensure completeness and spatial alignment of each decomposed\ninstance, we develop a video-diffusion-based amodal completion technique that\neffectively handles occlusions and shadows between objects, and introduce shape\nprior injection to ensure spatial coherence within the scene. "},{"type":"highlight","text":"Experimental\nresults demonstrate that our method produces more natural object arrangements\nand complete object instances suitable for interactive applications, while\nmaintaining physical plausibility and alignment with user inputs."}]},{"paper":{"id":"2510.13678","authors":[{"_id":"68f051ed36f8b025381e1902","name":"Xinyang Li","hidden":false},{"_id":"68f051ed36f8b025381e1903","name":"Tengfei Wang","hidden":false},{"_id":"68f051ed36f8b025381e1904","user":{"_id":"64b4b415a15d33a1bcc6ba6a","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64b4b415a15d33a1bcc6ba6a/5U2tvq8joAHsJEsqA3NJ6.jpeg","isPro":false,"fullname":"Zixiao Gu","user":"nightkiller","type":"user","name":"nightkiller"},"name":"Zixiao Gu","status":"claimed_verified","statusLastChangedAt":"2025-10-16T06:39:05.791Z","hidden":false},{"_id":"68f051ed36f8b025381e1905","name":"Shengchuan Zhang","hidden":false},{"_id":"68f051ed36f8b025381e1906","name":"Chunchao Guo","hidden":false},{"_id":"68f051ed36f8b025381e1907","name":"Liujuan Cao","hidden":false}],"publishedAt":"2025-10-15T15:35:48.000Z","title":"FlashWorld: High-quality 3D Scene Generation within Seconds","summary":"We propose FlashWorld, a generative model that produces 3D scenes from a\nsingle image or text prompt in seconds, 10~100times faster than previous\nworks while possessing superior rendering quality. Our approach shifts from the\nconventional multi-view-oriented (MV-oriented) paradigm, which generates\nmulti-view images for subsequent 3D reconstruction, to a 3D-oriented approach\nwhere the model directly produces 3D Gaussian representations during multi-view\ngeneration. While ensuring 3D consistency, 3D-oriented method typically suffers\npoor visual quality. FlashWorld includes a dual-mode pre-training phase\nfollowed by a cross-mode post-training phase, effectively integrating the\nstrengths of both paradigms. Specifically, leveraging the prior from a video\ndiffusion model, we first pre-train a dual-mode multi-view diffusion model,\nwhich jointly supports MV-oriented and 3D-oriented generation modes. To bridge\nthe quality gap in 3D-oriented generation, we further propose a cross-mode\npost-training distillation by matching distribution from consistent 3D-oriented\nmode to high-quality MV-oriented mode. This not only enhances visual quality\nwhile maintaining 3D consistency, but also reduces the required denoising steps\nfor inference. Also, we propose a strategy to leverage massive single-view\nimages and text prompts during this process to enhance the model's\ngeneralization to out-of-distribution inputs. Extensive experiments demonstrate\nthe superiority and efficiency of our method.","upvotes":74,"discussionId":"68f051ed36f8b025381e1908","projectPage":"https://imlixinyang.github.io/FlashWorld-Project-Page/","githubRepo":"https://github.com/imlixinyang/FlashWorld","githubRepoAddedBy":"user","ai_summary":"FlashWorld generates 3D scenes from single images or text prompts quickly and with high quality by combining MV-oriented and 3D-oriented generation methods.","ai_keywords":["generative model","3D scenes","single image","text prompt","3D Gaussian representations","MV-oriented paradigm","3D-oriented approach","dual-mode pre-training","cross-mode post-training","video diffusion model","multi-view diffusion model","cross-mode post-training distillation","denoising steps","single-view images","out-of-distribution inputs"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":858},"publishedAt":"2025-10-15T11:35:48.000Z","title":"FlashWorld: High-quality 3D Scene Generation within Seconds","summary":"We propose FlashWorld, a generative model that produces 3D scenes from a\nsingle image or text prompt in seconds, 10~100times faster than previous\nworks while possessing superior rendering quality. Our approach shifts from the\nconventional multi-view-oriented (MV-oriented) paradigm, which generates\nmulti-view images for subsequent 3D reconstruction, to a 3D-oriented approach\nwhere the model directly produces 3D Gaussian representations during multi-view\ngeneration. While ensuring 3D consistency, 3D-oriented method typically suffers\npoor visual quality. FlashWorld includes a dual-mode pre-training phase\nfollowed by a cross-mode post-training phase, effectively integrating the\nstrengths of both paradigms. Specifically, leveraging the prior from a video\ndiffusion model, we first pre-train a dual-mode multi-view diffusion model,\nwhich jointly supports MV-oriented and 3D-oriented generation modes. To bridge\nthe quality gap in 3D-oriented generation, we further propose a cross-mode\npost-training distillation by matching distribution from consistent 3D-oriented\nmode to high-quality MV-oriented mode. This not only enhances visual quality\nwhile maintaining 3D consistency, but also reduces the required denoising steps\nfor inference. Also, we propose a strategy to leverage massive single-view\nimages and text prompts during this process to enhance the model's\ngeneralization to out-of-distribution inputs. Extensive experiments demonstrate\nthe superiority and efficiency of our method.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2510.13678.png","numComments":2,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"FlashWorld: High-quality 3D Scene Generation within Seconds"}],"highlightedSummary":[{"type":"text","text":"We propose FlashWorld, a generative model that produces 3D scenes from a\nsingle image or text prompt in seconds, 10~100times faster than previous\nworks while possessing superior rendering quality. Our approach shifts from the\nconventional multi-view-oriented (MV-oriented) paradigm, which generates\nmulti-view images for subsequent 3D reconstruction, to a 3D-oriented approach\nwhere the model directly produces 3D Gaussian representations during multi-view\ngeneration. While ensuring 3D consistency, 3D-oriented method typically suffers\npoor visual quality. FlashWorld includes a dual-mode pre-training phase\nfollowed by a cross-mode post-training phase, effectively integrating the\nstrengths of both paradigms. Specifically, leveraging the prior from a video\ndiffusion model, we first pre-train a dual-mode multi-view diffusion model,\nwhich jointly supports MV-oriented and 3D-oriented generation modes. To bridge\nthe quality gap in 3D-oriented generation, we further propose a cross-mode\npost-training distillation by matching distribution from consistent 3D-oriented\nmode to high-quality MV-oriented mode. This not only enhances visual quality\nwhile maintaining 3D consistency, but also reduces the required denoising steps\nfor inference. Also, we propose a strategy to leverage massive single-view\nimages and text prompts during this process to enhance the model's\ngeneralization to out-of-distribution inputs. Extensive experiments demonstrate\nthe superiority and efficiency of our method."}]},{"paper":{"id":"2007.05756","authors":[{"_id":"67a7949f868e3124b738d9ac","user":{"_id":"63e16165f0039731dfdd442a","avatarUrl":"/avatars/4e7e560aa9b5cb0ae032a2bfbbe9a331.svg","isPro":false,"fullname":"Boris Knyazev","user":"bknyaz","type":"user","name":"bknyaz"},"name":"Boris Knyazev","status":"claimed_verified","statusLastChangedAt":"2025-02-24T09:25:38.400Z","hidden":false},{"_id":"67a7949f868e3124b738d9ad","name":"Harm de Vries","hidden":false},{"_id":"67a7949f868e3124b738d9ae","name":"Cătălina Cangea","hidden":false},{"_id":"67a7949f868e3124b738d9af","name":"Graham W. Taylor","hidden":false},{"_id":"67a7949f868e3124b738d9b0","name":"Aaron Courville","hidden":false},{"_id":"67a7949f868e3124b738d9b1","name":"Eugene Belilovsky","hidden":false}],"publishedAt":"2020-07-11T12:11:53.000Z","title":"Generative Compositional Augmentations for Scene Graph Prediction","summary":"Inferring objects and their relationships from an image in the form of a\nscene graph is useful in many applications at the intersection of vision and\nlanguage. We consider a challenging problem of compositional generalization\nthat emerges in this task due to a long tail data distribution. Current scene\ngraph generation models are trained on a tiny fraction of the distribution\ncorresponding to the most frequent compositions, e.g. <cup, on, table>.\nHowever, test images might contain zero- and few-shot compositions of objects\nand relationships, e.g. <cup, on, surfboard>. Despite each of the object\ncategories and the predicate (e.g. 'on') being frequent in the training data,\nthe models often fail to properly understand such unseen or rare compositions.\nTo improve generalization, it is natural to attempt increasing the diversity of\nthe training distribution. However, in the graph domain this is non-trivial. To\nthat end, we propose a method to synthesize rare yet plausible scene graphs by\nperturbing real ones. We then propose and empirically study a model based on\nconditional generative adversarial networks (GANs) that allows us to generate\nvisual features of perturbed scene graphs and learn from them in a joint\nfashion. When evaluated on the Visual Genome dataset, our approach yields\nmarginal, but consistent improvements in zero- and few-shot metrics. We analyze\nthe limitations of our approach indicating promising directions for future\nresearch.","upvotes":0,"discussionId":"67a794a1868e3124b738da06","ai_summary":"A conditional GAN-based approach improves zero- and few-shot generalization in scene graph generation by synthesizing rare yet plausible scene graphs through perturbation.","ai_keywords":["scene graph generation","compositional generalization","long tail data distribution","conditional generative adversarial networks (GANs)","Visual Genome dataset","synthetic scene graphs","perturbation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2020-07-11T08:11:53.000Z","title":"Generative Compositional Augmentations for Scene Graph Prediction","summary":"Inferring objects and their relationships from an image in the form of a\nscene graph is useful in many applications at the intersection of vision and\nlanguage. We consider a challenging problem of compositional generalization\nthat emerges in this task due to a long tail data distribution. Current scene\ngraph generation models are trained on a tiny fraction of the distribution\ncorresponding to the most frequent compositions, e.g. <cup, on, table>.\nHowever, test images might contain zero- and few-shot compositions of objects\nand relationships, e.g. <cup, on, surfboard>. Despite each of the object\ncategories and the predicate (e.g. 'on') being frequent in the training data,\nthe models often fail to properly understand such unseen or rare compositions.\nTo improve generalization, it is natural to attempt increasing the diversity of\nthe training distribution. However, in the graph domain this is non-trivial. To\nthat end, we propose a method to synthesize rare yet plausible scene graphs by\nperturbing real ones. We then propose and empirically study a model based on\nconditional generative adversarial networks (GANs) that allows us to generate\nvisual features of perturbed scene graphs and learn from them in a joint\nfashion. When evaluated on the Visual Genome dataset, our approach yields\nmarginal, but consistent improvements in zero- and few-shot metrics. We analyze\nthe limitations of our approach indicating promising directions for future\nresearch.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2007.05756.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Generative Compositional Augmentations for Scene Graph Prediction"}],"highlightedSummary":[{"type":"text","text":"Inferring objects and their relationships from an image in the form of a\nscene graph is useful in many applications at the intersection of vision and\nlanguage. We consider a challenging problem of compositional generalization\nthat emerges in this task due to a long tail data distribution. "},{"type":"highlight","text":"Current scene\ngraph generation models are trained on a tiny fraction of the distribution\ncorresponding to the most frequent compositions, e.g."},{"type":"text","text":" <cup, on, table>.\nHowever, test images might contain zero- and few-shot compositions of objects\nand relationships, e.g. <cup, on, surfboard>. Despite each of the object\ncategories and the predicate (e.g. 'on') being frequent in the training data,\nthe models often fail to properly understand such unseen or rare compositions.\nTo improve generalization, it is natural to attempt increasing the diversity of\nthe training distribution. However, in the graph domain this is non-trivial. To\nthat end, we propose a method to synthesize rare yet plausible scene graphs by\nperturbing real ones. We then propose and empirically study a model based on\nconditional generative adversarial networks (GANs) that allows us to generate\nvisual features of perturbed scene graphs and learn from them in a joint\nfashion. When evaluated on the Visual Genome dataset, our approach yields\nmarginal, but consistent improvements in zero- and few-shot metrics. We analyze\nthe limitations of our approach indicating promising directions for future\nresearch."}]},{"paper":{"id":"2501.03059","authors":[{"_id":"677cd67175191fdc79cbcc04","user":{"_id":"646d239f4220471ca0c6471c","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/646d239f4220471ca0c6471c/sRwzko8XEUVCkeD7jXceH.jpeg","isPro":false,"fullname":"Guy Yariv","user":"GuyYariv","type":"user","name":"GuyYariv"},"name":"Guy Yariv","status":"claimed_verified","statusLastChangedAt":"2025-01-07T09:37:35.128Z","hidden":false},{"_id":"677cd67175191fdc79cbcc05","name":"Yuval Kirstain","hidden":false},{"_id":"677cd67175191fdc79cbcc06","user":{"_id":"64d8cb34505306fcd2fb89c3","avatarUrl":"/avatars/4ae9ffe5d494a7d53bc519ff8e402b3e.svg","isPro":false,"fullname":"Amit Zohar","user":"amitz","type":"user","name":"amitz"},"name":"Amit Zohar","status":"admin_assigned","statusLastChangedAt":"2025-01-07T13:17:40.162Z","hidden":false},{"_id":"677cd67175191fdc79cbcc07","user":{"_id":"63468e6ca6f101c7f9049132","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1665662111525-63468e6ca6f101c7f9049132.png","isPro":false,"fullname":"Shelly Sheynin","user":"shellysheynin","type":"user","name":"shellysheynin"},"name":"Shelly Sheynin","status":"admin_assigned","statusLastChangedAt":"2025-01-07T13:17:46.692Z","hidden":false},{"_id":"677cd67175191fdc79cbcc08","name":"Yaniv Taigman","hidden":false},{"_id":"677cd67175191fdc79cbcc09","user":{"_id":"6481e135578646b5c2386728","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6481e135578646b5c2386728/SPva4iNw0pORiCXD45cx9.jpeg","isPro":false,"fullname":"Yossi Adi","user":"adiyoss","type":"user","name":"adiyoss"},"name":"Yossi Adi","status":"admin_assigned","statusLastChangedAt":"2025-01-07T13:17:56.992Z","hidden":false},{"_id":"677cd67175191fdc79cbcc0a","user":{"_id":"6345a9b9a8c2ff9f1377faab","avatarUrl":"/avatars/a5f2b999ef8b967b2af9f41afcd9d475.svg","isPro":false,"fullname":"Sagie Benaim","user":"sagiebenaim","type":"user","name":"sagiebenaim"},"name":"Sagie Benaim","status":"admin_assigned","statusLastChangedAt":"2025-01-07T13:18:03.458Z","hidden":false},{"_id":"677cd67175191fdc79cbcc0b","user":{"_id":"6304d514dae2eb7d08413d62","avatarUrl":"/avatars/02a571bc791b78d3993d9a0484b70a29.svg","isPro":false,"fullname":"Adam Polyak","user":"adampo","type":"user","name":"adampo"},"name":"Adam Polyak","status":"admin_assigned","statusLastChangedAt":"2025-01-07T13:18:10.411Z","hidden":false}],"publishedAt":"2025-01-06T14:49:26.000Z","title":"Through-The-Mask: Mask-based Motion Trajectories for Image-to-Video\n  Generation","summary":"We consider the task of Image-to-Video (I2V) generation, which involves\ntransforming static images into realistic video sequences based on a textual\ndescription. While recent advancements produce photorealistic outputs, they\nfrequently struggle to create videos with accurate and consistent object\nmotion, especially in multi-object scenarios. To address these limitations, we\npropose a two-stage compositional framework that decomposes I2V generation\ninto: (i) An explicit intermediate representation generation stage, followed by\n(ii) A video generation stage that is conditioned on this representation. Our\nkey innovation is the introduction of a mask-based motion trajectory as an\nintermediate representation, that captures both semantic object information and\nmotion, enabling an expressive but compact representation of motion and\nsemantics. To incorporate the learned representation in the second stage, we\nutilize object-level attention objectives. Specifically, we consider a spatial,\nper-object, masked-cross attention objective, integrating object-specific\nprompts into corresponding latent space regions and a masked spatio-temporal\nself-attention objective, ensuring frame-to-frame consistency for each object.\nWe evaluate our method on challenging benchmarks with multi-object and\nhigh-motion scenarios and empirically demonstrate that the proposed method\nachieves state-of-the-art results in temporal coherence, motion realism, and\ntext-prompt faithfulness. Additionally, we introduce \\benchmark, a new\nchallenging benchmark for single-object and multi-object I2V generation, and\ndemonstrate our method's superiority on this benchmark. Project page is\navailable at https://guyyariv.github.io/TTM/.","upvotes":22,"discussionId":"677cd67375191fdc79cbcc76","projectPage":"https://guyyariv.github.io/TTM/","ai_summary":"The proposed two-stage framework for Image-to-Video generation uses mask-based motion trajectories and object-level attention to produce realistic and semantically consistent videos from text descriptions.","ai_keywords":["image-to-video generation","intermediate representation","mask-based motion trajectory","object-level attention","masked-cross attention","masked spatio-temporal self-attention","temporal coherence","motion realism","text-prompt faithfulness"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-01-06T09:49:26.000Z","title":"Through-The-Mask: Mask-based Motion Trajectories for Image-to-Video\n  Generation","summary":"We consider the task of Image-to-Video (I2V) generation, which involves\ntransforming static images into realistic video sequences based on a textual\ndescription. While recent advancements produce photorealistic outputs, they\nfrequently struggle to create videos with accurate and consistent object\nmotion, especially in multi-object scenarios. To address these limitations, we\npropose a two-stage compositional framework that decomposes I2V generation\ninto: (i) An explicit intermediate representation generation stage, followed by\n(ii) A video generation stage that is conditioned on this representation. Our\nkey innovation is the introduction of a mask-based motion trajectory as an\nintermediate representation, that captures both semantic object information and\nmotion, enabling an expressive but compact representation of motion and\nsemantics. To incorporate the learned representation in the second stage, we\nutilize object-level attention objectives. Specifically, we consider a spatial,\nper-object, masked-cross attention objective, integrating object-specific\nprompts into corresponding latent space regions and a masked spatio-temporal\nself-attention objective, ensuring frame-to-frame consistency for each object.\nWe evaluate our method on challenging benchmarks with multi-object and\nhigh-motion scenarios and empirically demonstrate that the proposed method\nachieves state-of-the-art results in temporal coherence, motion realism, and\ntext-prompt faithfulness. Additionally, we introduce \\benchmark, a new\nchallenging benchmark for single-object and multi-object I2V generation, and\ndemonstrate our method's superiority on this benchmark. Project page is\navailable at https://guyyariv.github.io/TTM/.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2501.03059.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Through-The-Mask: Mask-based Motion Trajectories for Image-to-Video\n  Generation"}],"highlightedSummary":[{"type":"text","text":"We consider the task of Image-to-Video (I2V) generation, which involves\ntransforming static images into realistic video sequences based on a textual\ndescription. While recent advancements produce photorealistic outputs, they\nfrequently struggle to create videos with accurate and consistent object\nmotion, especially in multi-object scenarios. "},{"type":"highlight","text":"To address these limitations, we\npropose a two-stage compositional framework that decomposes I2V generation\ninto: (i) An explicit intermediate representation generation stage, followed by\n(ii) A video generation stage that is conditioned on this representation."},{"type":"text","text":" Our\nkey innovation is the introduction of a mask-based motion trajectory as an\nintermediate representation, that captures both semantic object information and\nmotion, enabling an expressive but compact representation of motion and\nsemantics. To incorporate the learned representation in the second stage, we\nutilize object-level attention objectives. Specifically, we consider a spatial,\nper-object, masked-cross attention objective, integrating object-specific\nprompts into corresponding latent space regions and a masked spatio-temporal\nself-attention objective, ensuring frame-to-frame consistency for each object.\nWe evaluate our method on challenging benchmarks with multi-object and\nhigh-motion scenarios and empirically demonstrate that the proposed method\nachieves state-of-the-art results in temporal coherence, motion realism, and\ntext-prompt faithfulness. Additionally, we introduce \\benchmark, a new\nchallenging benchmark for single-object and multi-object I2V generation, and\ndemonstrate our method's superiority on this benchmark. Project page is\navailable at https://guyyariv.github.io/TTM/."}]},{"paper":{"id":"2503.00513","authors":[{"_id":"67c952ff28d1ce7f8d9f79c7","user":{"_id":"652d3023cdb2a91205709b6a","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/652d3023cdb2a91205709b6a/x-MaRkBwSdZ0xIYDJg09u.jpeg","isPro":false,"fullname":"Hanxun Yu","user":"JonnyYu828","type":"user","name":"JonnyYu828"},"name":"Hanxun Yu","status":"claimed_verified","statusLastChangedAt":"2026-02-02T17:05:14.490Z","hidden":false},{"_id":"67c952ff28d1ce7f8d9f79c8","user":{"_id":"64c48a78d07620bdc99777d4","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64c48a78d07620bdc99777d4/NJC4Ot0a7YSdU5RC6dgga.jpeg","isPro":false,"fullname":"LI WENTONG","user":"sunshine-lwt","type":"user","name":"sunshine-lwt"},"name":"Wentong Li","status":"claimed_verified","statusLastChangedAt":"2025-11-18T14:13:25.688Z","hidden":false},{"_id":"67c952ff28d1ce7f8d9f79c9","user":{"_id":"66863d26e2b71e3d09189ae9","avatarUrl":"/avatars/3c0e6f30e053f2e622ae75e1dc43edba.svg","isPro":false,"fullname":"Song Wang","user":"songw-zju","type":"user","name":"songw-zju"},"name":"Song Wang","status":"claimed_verified","statusLastChangedAt":"2025-06-27T08:58:29.480Z","hidden":false},{"_id":"67c952ff28d1ce7f8d9f79ca","name":"Junbo Chen","hidden":false},{"_id":"67c952ff28d1ce7f8d9f79cb","name":"Jianke Zhu","hidden":false}],"publishedAt":"2025-03-01T14:38:42.000Z","title":"Inst3D-LMM: Instance-Aware 3D Scene Understanding with Multi-modal\n  Instruction Tuning","summary":"Despite encouraging progress in 3D scene understanding, it remains\nchallenging to develop an effective Large Multi-modal Model (LMM) that is\ncapable of understanding and reasoning in complex 3D environments. Most\nprevious methods typically encode 3D point and 2D image features separately,\nneglecting interactions between 2D semantics and 3D object properties, as well\nas the spatial relationships within the 3D environment. This limitation not\nonly hinders comprehensive representations of 3D scene, but also compromises\ntraining and inference efficiency. To address these challenges, we propose a\nunified Instance-aware 3D Large Multi-modal Model (Inst3D-LMM) to deal with\nmultiple 3D scene understanding tasks simultaneously. To obtain the\nfine-grained instance-level visual tokens, we first introduce a novel\nMulti-view Cross-Modal Fusion (MCMF) module to inject the multi-view 2D\nsemantics into their corresponding 3D geometric features. For scene-level\nrelation-aware tokens, we further present a 3D Instance Spatial Relation\n(3D-ISR) module to capture the intricate pairwise spatial relationships among\nobjects. Additionally, we perform end-to-end multi-task instruction tuning\nsimultaneously without the subsequent task-specific fine-tuning. Extensive\nexperiments demonstrate that our approach outperforms the state-of-the-art\nmethods across 3D scene understanding, reasoning and grounding tasks. Source\ncode is available at https://github.com/hanxunyu/Inst3D-LMM","upvotes":2,"discussionId":"67c9530028d1ce7f8d9f79f9","githubRepo":"https://github.com/hanxunyu/Inst3D-LMM","githubRepoAddedBy":"user","ai_summary":"Inst3D-LMM uses MCMF and 3D-ISR to enhance 3D scene understanding and reasoning by integrating 2D semantics and 3D geometry, achieving superior performance in multi-task instruction tuning.","ai_keywords":["Large Multi-modal Model","3D scene understanding","3D point features","2D image features","Multi-view Cross-Modal Fusion","3D Instance Spatial Relation","Instance-aware","end-to-end multi-task instruction tuning","3D scene understanding tasks","reasoning","grounding tasks"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":134},"publishedAt":"2025-03-01T09:38:42.000Z","title":"Inst3D-LMM: Instance-Aware 3D Scene Understanding with Multi-modal\n  Instruction Tuning","summary":"Despite encouraging progress in 3D scene understanding, it remains\nchallenging to develop an effective Large Multi-modal Model (LMM) that is\ncapable of understanding and reasoning in complex 3D environments. Most\nprevious methods typically encode 3D point and 2D image features separately,\nneglecting interactions between 2D semantics and 3D object properties, as well\nas the spatial relationships within the 3D environment. This limitation not\nonly hinders comprehensive representations of 3D scene, but also compromises\ntraining and inference efficiency. To address these challenges, we propose a\nunified Instance-aware 3D Large Multi-modal Model (Inst3D-LMM) to deal with\nmultiple 3D scene understanding tasks simultaneously. To obtain the\nfine-grained instance-level visual tokens, we first introduce a novel\nMulti-view Cross-Modal Fusion (MCMF) module to inject the multi-view 2D\nsemantics into their corresponding 3D geometric features. For scene-level\nrelation-aware tokens, we further present a 3D Instance Spatial Relation\n(3D-ISR) module to capture the intricate pairwise spatial relationships among\nobjects. Additionally, we perform end-to-end multi-task instruction tuning\nsimultaneously without the subsequent task-specific fine-tuning. Extensive\nexperiments demonstrate that our approach outperforms the state-of-the-art\nmethods across 3D scene understanding, reasoning and grounding tasks. Source\ncode is available at https://github.com/hanxunyu/Inst3D-LMM","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2503.00513.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Inst3D-LMM: Instance-Aware 3D Scene Understanding with Multi-modal\n  Instruction Tuning"}],"highlightedSummary":[{"type":"text","text":"Despite encouraging progress in 3D scene understanding, it remains\nchallenging to develop an effective Large Multi-modal Model (LMM) that is\ncapable of understanding and reasoning in complex 3D environments. "},{"type":"highlight","text":"Most\nprevious methods typically encode 3D point and 2D image features separately,\nneglecting interactions between 2D semantics and 3D object properties, as well\nas the spatial relationships within the 3D environment."},{"type":"text","text":" This limitation not\nonly hinders comprehensive representations of 3D scene, but also compromises\ntraining and inference efficiency. To address these challenges, we propose a\nunified Instance-aware 3D Large Multi-modal Model (Inst3D-LMM) to deal with\nmultiple 3D scene understanding tasks simultaneously. To obtain the\nfine-grained instance-level visual tokens, we first introduce a novel\nMulti-view Cross-Modal Fusion (MCMF) module to inject the multi-view 2D\nsemantics into their corresponding 3D geometric features. For scene-level\nrelation-aware tokens, we further present a 3D Instance Spatial Relation\n(3D-ISR) module to capture the intricate pairwise spatial relationships among\nobjects. Additionally, we perform end-to-end multi-task instruction tuning\nsimultaneously without the subsequent task-specific fine-tuning. Extensive\nexperiments demonstrate that our approach outperforms the state-of-the-art\nmethods across 3D scene understanding, reasoning and grounding tasks. Source\ncode is available at https://github.com/hanxunyu/Inst3D-LMM"}]},{"paper":{"id":"2407.02329","authors":[{"_id":"6940ccd365f1e24a1177ffcd","name":"Dewei Zhou","hidden":false},{"_id":"6940ccd365f1e24a1177ffce","name":"You Li","hidden":false},{"_id":"6940ccd365f1e24a1177ffcf","name":"Fan Ma","hidden":false},{"_id":"6940ccd365f1e24a1177ffd0","name":"Zongxin Yang","hidden":false},{"_id":"6940ccd365f1e24a1177ffd1","name":"Yi Yang","hidden":false}],"publishedAt":"2024-07-02T14:59:37.000Z","title":"MIGC++: Advanced Multi-Instance Generation Controller for Image Synthesis","summary":"We introduce the Multi-Instance Generation (MIG) task, which focuses on generating multiple instances within a single image, each accurately placed at predefined positions with attributes such as category, color, and shape, strictly following user specifications. MIG faces three main challenges: avoiding attribute leakage between instances, supporting diverse instance descriptions, and maintaining consistency in iterative generation. To address attribute leakage, we propose the Multi-Instance Generation Controller (MIGC). MIGC generates multiple instances through a divide-and-conquer strategy, breaking down multi-instance shading into single-instance tasks with singular attributes, later integrated. To provide more types of instance descriptions, we developed MIGC++. MIGC++ allows attribute control through text \\& images and position control through boxes \\& masks. Lastly, we introduced the Consistent-MIG algorithm to enhance the iterative MIG ability of MIGC and MIGC++. This algorithm ensures consistency in unmodified regions during the addition, deletion, or modification of instances, and preserves the identity of instances when their attributes are changed. We introduce the COCO-MIG and Multimodal-MIG benchmarks to evaluate these methods. Extensive experiments on these benchmarks, along with the COCO-Position benchmark and DrawBench, demonstrate that our methods substantially outperform existing techniques, maintaining precise control over aspects including position, attribute, and quantity. Project page: https://github.com/limuloo/MIGC.","upvotes":0,"discussionId":"6940ccd365f1e24a1177ffd2","githubRepo":"https://github.com/limuloo/migc","githubRepoAddedBy":"auto","ai_summary":"The paper presents Multi-Instance Generation (MIG) for precise placement and attribute control of multiple instances within an image, addressing challenges like attribute leakage, diverse descriptions, and iterative consistency with the MIGC, MIGC++, and Consistent-MIG algorithms.","ai_keywords":["Multi-Instance Generation","MIG","Multi-Instance Generation Controller","MIGC","MIGC++","Consistent-MIG","attribute leakage","text","images","boxes","masks","COCO-MIG","Multimodal-MIG","COCO-Position benchmark","DrawBench"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":613},"publishedAt":"2024-07-02T10:59:37.000Z","title":"MIGC++: Advanced Multi-Instance Generation Controller for Image Synthesis","summary":"We introduce the Multi-Instance Generation (MIG) task, which focuses on generating multiple instances within a single image, each accurately placed at predefined positions with attributes such as category, color, and shape, strictly following user specifications. MIG faces three main challenges: avoiding attribute leakage between instances, supporting diverse instance descriptions, and maintaining consistency in iterative generation. To address attribute leakage, we propose the Multi-Instance Generation Controller (MIGC). MIGC generates multiple instances through a divide-and-conquer strategy, breaking down multi-instance shading into single-instance tasks with singular attributes, later integrated. To provide more types of instance descriptions, we developed MIGC++. MIGC++ allows attribute control through text \\& images and position control through boxes \\& masks. Lastly, we introduced the Consistent-MIG algorithm to enhance the iterative MIG ability of MIGC and MIGC++. This algorithm ensures consistency in unmodified regions during the addition, deletion, or modification of instances, and preserves the identity of instances when their attributes are changed. We introduce the COCO-MIG and Multimodal-MIG benchmarks to evaluate these methods. Extensive experiments on these benchmarks, along with the COCO-Position benchmark and DrawBench, demonstrate that our methods substantially outperform existing techniques, maintaining precise control over aspects including position, attribute, and quantity. Project page: https://github.com/limuloo/MIGC.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2407.02329.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"MIGC++: Advanced Multi-Instance Generation Controller for Image Synthesis"}],"highlightedSummary":[{"type":"highlight","text":"We introduce the Multi-Instance Generation (MIG) task, which focuses on generating multiple instances within a single image, each accurately placed at predefined positions with attributes such as category, color, and shape, strictly following user specifications."},{"type":"text","text":" MIG faces three main challenges: avoiding attribute leakage between instances, supporting diverse instance descriptions, and maintaining consistency in iterative generation. To address attribute leakage, we propose the Multi-Instance Generation Controller (MIGC). MIGC generates multiple instances through a divide-and-conquer strategy, breaking down multi-instance shading into single-instance tasks with singular attributes, later integrated. To provide more types of instance descriptions, we developed MIGC++. MIGC++ allows attribute control through text \\& images and position control through boxes \\& masks. Lastly, we introduced the Consistent-MIG algorithm to enhance the iterative MIG ability of MIGC and MIGC++. This algorithm ensures consistency in unmodified regions during the addition, deletion, or modification of instances, and preserves the identity of instances when their attributes are changed. We introduce the COCO-MIG and Multimodal-MIG benchmarks to evaluate these methods. Extensive experiments on these benchmarks, along with the COCO-Position benchmark and DrawBench, demonstrate that our methods substantially outperform existing techniques, maintaining precise control over aspects including position, attribute, and quantity. Project page: https://github.com/limuloo/MIGC."}]},{"paper":{"id":"2503.16611","authors":[{"_id":"67e182b63e646b97d8ef4a42","name":"Katja Schwarz","hidden":false},{"_id":"67e182b63e646b97d8ef4a43","name":"Denys Rozumnyi","hidden":false},{"_id":"67e182b63e646b97d8ef4a44","name":"Samuel Rota Bulò","hidden":false},{"_id":"67e182b63e646b97d8ef4a45","name":"Lorenzo Porzi","hidden":false},{"_id":"67e182b63e646b97d8ef4a46","name":"Peter Kontschieder","hidden":false}],"publishedAt":"2025-03-20T18:06:12.000Z","title":"A Recipe for Generating 3D Worlds From a Single Image","summary":"We introduce a recipe for generating immersive 3D worlds from a single image\nby framing the task as an in-context learning problem for 2D inpainting models.\nThis approach requires minimal training and uses existing generative models.\nOur process involves two steps: generating coherent panoramas using a\npre-trained diffusion model and lifting these into 3D with a metric depth\nestimator. We then fill unobserved regions by conditioning the inpainting model\non rendered point clouds, requiring minimal fine-tuning. Tested on both\nsynthetic and real images, our method produces high-quality 3D environments\nsuitable for VR display. By explicitly modeling the 3D structure of the\ngenerated environment from the start, our approach consistently outperforms\nstate-of-the-art, video synthesis-based methods along multiple quantitative\nimage quality metrics. Project Page: https://katjaschwarz.github.io/worlds/","upvotes":1,"discussionId":"67e182bd3e646b97d8ef4d49","projectPage":"https://katjaschwarz.github.io/worlds/","ai_summary":"A method for generating 3D environments from a single image using in-context learning with a diffusion model and depth estimation, followed by inpainting with point clouds, outperforms existing video synthesis methods.","ai_keywords":["in-context learning","diffusion model","metric depth estimator","inpainting model","point clouds"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-03-20T14:06:12.000Z","title":"A Recipe for Generating 3D Worlds From a Single Image","summary":"We introduce a recipe for generating immersive 3D worlds from a single image\nby framing the task as an in-context learning problem for 2D inpainting models.\nThis approach requires minimal training and uses existing generative models.\nOur process involves two steps: generating coherent panoramas using a\npre-trained diffusion model and lifting these into 3D with a metric depth\nestimator. We then fill unobserved regions by conditioning the inpainting model\non rendered point clouds, requiring minimal fine-tuning. Tested on both\nsynthetic and real images, our method produces high-quality 3D environments\nsuitable for VR display. By explicitly modeling the 3D structure of the\ngenerated environment from the start, our approach consistently outperforms\nstate-of-the-art, video synthesis-based methods along multiple quantitative\nimage quality metrics. Project Page: https://katjaschwarz.github.io/worlds/","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2503.16611.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"A Recipe for Generating 3D Worlds From a Single Image"}],"highlightedSummary":[{"type":"highlight","text":"We introduce a recipe for generating immersive 3D worlds from a single image\nby framing the task as an in-context learning problem for 2D inpainting models."},{"type":"text","text":"\nThis approach requires minimal training and uses existing generative models.\nOur process involves two steps: generating coherent panoramas using a\npre-trained diffusion model and lifting these into 3D with a metric depth\nestimator. We then fill unobserved regions by conditioning the inpainting model\non rendered point clouds, requiring minimal fine-tuning. Tested on both\nsynthetic and real images, our method produces high-quality 3D environments\nsuitable for VR display. By explicitly modeling the 3D structure of the\ngenerated environment from the start, our approach consistently outperforms\nstate-of-the-art, video synthesis-based methods along multiple quantitative\nimage quality metrics. Project Page: https://katjaschwarz.github.io/worlds/"}]},{"paper":{"id":"2406.07472","authors":[{"_id":"6669056f61588d66710d13ba","name":"Heng Yu","hidden":false},{"_id":"6669056f61588d66710d13bb","user":{"_id":"64c166de11f3f1d23a99c6fd","avatarUrl":"/avatars/d4730f70a6f1cb51266862c7a8d54f77.svg","isPro":false,"fullname":"Chaoyang Wang","user":"cwang9","type":"user","name":"cwang9"},"name":"Chaoyang Wang","status":"admin_assigned","statusLastChangedAt":"2024-06-12T08:17:01.607Z","hidden":false},{"_id":"6669056f61588d66710d13bc","user":{"_id":"63e473cf9db5da2dc1f4e9ff","avatarUrl":"/avatars/6973b4c959ef8bb72c40c611d384a78a.svg","isPro":false,"fullname":"peiye zhuang","user":"Kelest","type":"user","name":"Kelest"},"name":"Peiye Zhuang","status":"admin_assigned","statusLastChangedAt":"2024-06-12T08:17:07.735Z","hidden":false},{"_id":"6669056f61588d66710d13bd","user":{"_id":"6315358a362e3e95ea538081","avatarUrl":"/avatars/3b089a25a87c2e83c6b23ccb5d2dc73e.svg","isPro":false,"fullname":"Willi Menapace","user":"willi-menapace","type":"user","name":"willi-menapace"},"name":"Willi Menapace","status":"admin_assigned","statusLastChangedAt":"2024-06-12T08:17:14.379Z","hidden":false},{"_id":"6669056f61588d66710d13be","user":{"_id":"64276311eb9a0ed86180715b","avatarUrl":"/avatars/76f933cd549f10e5e2db379de235d304.svg","isPro":false,"fullname":"Aliaksandr Siarohin","user":"aliaksandr-siarohin","type":"user","name":"aliaksandr-siarohin"},"name":"Aliaksandr Siarohin","status":"admin_assigned","statusLastChangedAt":"2024-06-12T08:17:23.361Z","hidden":false},{"_id":"6669056f61588d66710d13bf","user":{"_id":"63f54aa73aa49d8cb97b84bc","avatarUrl":"/avatars/c73c5870039611ab9162daad46a1ba20.svg","isPro":false,"fullname":"junli cao","user":"jlcao2","type":"user","name":"jlcao2"},"name":"Junli Cao","status":"admin_assigned","statusLastChangedAt":"2024-06-12T08:17:29.415Z","hidden":false},{"_id":"6669056f61588d66710d13c0","user":{"_id":"64bbd2f865b648b2df859bc6","avatarUrl":"/avatars/698d98d8b42c478a3d7584b93c6dbcc2.svg","isPro":false,"fullname":"Laszlo Jeni","user":"laszlo-ai","type":"user","name":"laszlo-ai"},"name":"Laszlo A Jeni","status":"admin_assigned","statusLastChangedAt":"2024-06-12T08:17:39.866Z","hidden":false},{"_id":"6669056f61588d66710d13c1","name":"Sergey Tulyakov","hidden":false},{"_id":"6669056f61588d66710d13c2","user":{"_id":"631bbd918de8e645af71eed9","avatarUrl":"/avatars/1cb94deadc288d07c571c9289085548c.svg","isPro":false,"fullname":"Hsin-Ying Lee","user":"james371507","type":"user","name":"james371507"},"name":"Hsin-Ying Lee","status":"admin_assigned","statusLastChangedAt":"2024-06-12T08:17:54.621Z","hidden":false}],"publishedAt":"2024-06-11T17:19:26.000Z","title":"4Real: Towards Photorealistic 4D Scene Generation via Video Diffusion\n  Models","summary":"Existing dynamic scene generation methods mostly rely on distilling knowledge\nfrom pre-trained 3D generative models, which are typically fine-tuned on\nsynthetic object datasets. As a result, the generated scenes are often\nobject-centric and lack photorealism. To address these limitations, we\nintroduce a novel pipeline designed for photorealistic text-to-4D scene\ngeneration, discarding the dependency on multi-view generative models and\ninstead fully utilizing video generative models trained on diverse real-world\ndatasets. Our method begins by generating a reference video using the video\ngeneration model. We then learn the canonical 3D representation of the video\nusing a freeze-time video, delicately generated from the reference video. To\nhandle inconsistencies in the freeze-time video, we jointly learn a per-frame\ndeformation to model these imperfections. We then learn the temporal\ndeformation based on the canonical representation to capture dynamic\ninteractions in the reference video. The pipeline facilitates the generation of\ndynamic scenes with enhanced photorealism and structural integrity, viewable\nfrom multiple perspectives, thereby setting a new standard in 4D scene\ngeneration.","upvotes":13,"discussionId":"6669057261588d66710d1493","ai_summary":"A novel pipeline for photorealistic text-to-4D scene generation uses video generative models and learns canonical 3D and temporal deformations to enhance photorealism and structural integrity.","ai_keywords":["3D generative models","video generative models","freeze-time video","per-frame deformation","temporal deformation","4D scene generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-06-11T13:19:26.000Z","title":"4Real: Towards Photorealistic 4D Scene Generation via Video Diffusion\n  Models","summary":"Existing dynamic scene generation methods mostly rely on distilling knowledge\nfrom pre-trained 3D generative models, which are typically fine-tuned on\nsynthetic object datasets. As a result, the generated scenes are often\nobject-centric and lack photorealism. To address these limitations, we\nintroduce a novel pipeline designed for photorealistic text-to-4D scene\ngeneration, discarding the dependency on multi-view generative models and\ninstead fully utilizing video generative models trained on diverse real-world\ndatasets. Our method begins by generating a reference video using the video\ngeneration model. We then learn the canonical 3D representation of the video\nusing a freeze-time video, delicately generated from the reference video. To\nhandle inconsistencies in the freeze-time video, we jointly learn a per-frame\ndeformation to model these imperfections. We then learn the temporal\ndeformation based on the canonical representation to capture dynamic\ninteractions in the reference video. The pipeline facilitates the generation of\ndynamic scenes with enhanced photorealism and structural integrity, viewable\nfrom multiple perspectives, thereby setting a new standard in 4D scene\ngeneration.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2406.07472.png","numComments":3,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"4Real: Towards Photorealistic 4D Scene Generation via Video Diffusion\n  Models"}],"highlightedSummary":[{"type":"text","text":"Existing dynamic scene generation methods mostly rely on distilling knowledge\nfrom pre-trained 3D generative models, which are typically fine-tuned on\nsynthetic object datasets. As a result, the generated scenes are often\nobject-centric and lack photorealism. "},{"type":"highlight","text":"To address these limitations, we\nintroduce a novel pipeline designed for photorealistic text-to-4D scene\ngeneration, discarding the dependency on multi-view generative models and\ninstead fully utilizing video generative models trained on diverse real-world\ndatasets."},{"type":"text","text":" Our method begins by generating a reference video using the video\ngeneration model. We then learn the canonical 3D representation of the video\nusing a freeze-time video, delicately generated from the reference video. To\nhandle inconsistencies in the freeze-time video, we jointly learn a per-frame\ndeformation to model these imperfections. We then learn the temporal\ndeformation based on the canonical representation to capture dynamic\ninteractions in the reference video. The pipeline facilitates the generation of\ndynamic scenes with enhanced photorealism and structural integrity, viewable\nfrom multiple perspectives, thereby setting a new standard in 4D scene\ngeneration."}]},{"paper":{"id":"2602.19766","authors":[{"_id":"69cbc8435bd847a9dd4774aa","name":"Pengfei Wang","hidden":false},{"_id":"69cbc8435bd847a9dd4774ab","name":"Liyi Chen","hidden":false},{"_id":"69cbc8435bd847a9dd4774ac","user":{"_id":"6322d6d8510aa62b8125d755","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6322d6d8510aa62b8125d755/z3qvVBy01iItET9lMmum6.jpeg","isPro":false,"fullname":"Zhiyuan Ma","user":"zhizhi111","type":"user","name":"zhizhi111"},"name":"Zhiyuan Ma","status":"claimed_verified","statusLastChangedAt":"2026-06-29T13:25:42.237Z","hidden":false},{"_id":"69cbc8435bd847a9dd4774ad","name":"Yanjun Guo","hidden":false},{"_id":"69cbc8435bd847a9dd4774ae","name":"Guowen Zhang","hidden":false},{"_id":"69cbc8435bd847a9dd4774af","name":"Lei Zhang","hidden":false}],"publishedAt":"2026-02-23T12:15:54.000Z","title":"One2Scene: Geometric Consistent Explorable 3D Scene Generation from a Single Image","summary":"Generating explorable 3D scenes from a single image is a highly challenging problem in 3D vision. Existing methods struggle to support free exploration, often producing severe geometric distortions and noisy artifacts when the viewpoint moves far from the original perspective. We introduce One2Scene, an effective framework that decomposes this ill-posed problem into three tractable sub-tasks to enable immersive explorable scene generation. We first use a panorama generator to produce anchor views from a single input image as initialization. Then, we lift these 2D anchors into an explicit 3D geometric scaffold via a generalizable, feed-forward Gaussian Splatting network. Instead of treating the panorama as a single image for reconstruction, we project it into multiple sparse anchor views and reformulate the reconstruction task as multi-view stereo matching, which allows us to leverage robust geometric priors learned from large-scale multi-view datasets. A bidirectional feature fusion module is used to enforce cross-view consistency, yielding an efficient and geometrically reliable scaffold. Finally, the scaffold serves as a strong prior for a novel view generator to produce photorealistic and geometrically accurate views at arbitrary cameras. By explicitly conditioning on a 3D-consistent scaffold to perform reconstruction, One2Scene works stably under large camera motions, supporting immersive scene exploration. Extensive experiments show that One2Scene substantially outperforms state-of-the-art methods in panorama depth estimation, feed-forward 360° reconstruction, and explorable 3D scene generation. Project page: https://one2scene5406.github.io/","upvotes":0,"discussionId":"69cbc8435bd847a9dd4774b0","ai_summary":"One2Scene enables immersive 3D scene exploration by decomposing single-image generation into panorama generation, 3D scaffold construction using Gaussian Splatting, and multi-view stereo matching with bidirectional feature fusion.","ai_keywords":["panorama generator","Gaussian Splatting network","multi-view stereo matching","bidirectional feature fusion","3D geometric scaffold","novel view generator","photorealistic rendering","camera motion","geometric consistency"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2026-02-23T07:15:54.000Z","title":"One2Scene: Geometric Consistent Explorable 3D Scene Generation from a Single Image","summary":"Generating explorable 3D scenes from a single image is a highly challenging problem in 3D vision. Existing methods struggle to support free exploration, often producing severe geometric distortions and noisy artifacts when the viewpoint moves far from the original perspective. We introduce One2Scene, an effective framework that decomposes this ill-posed problem into three tractable sub-tasks to enable immersive explorable scene generation. We first use a panorama generator to produce anchor views from a single input image as initialization. Then, we lift these 2D anchors into an explicit 3D geometric scaffold via a generalizable, feed-forward Gaussian Splatting network. Instead of treating the panorama as a single image for reconstruction, we project it into multiple sparse anchor views and reformulate the reconstruction task as multi-view stereo matching, which allows us to leverage robust geometric priors learned from large-scale multi-view datasets. A bidirectional feature fusion module is used to enforce cross-view consistency, yielding an efficient and geometrically reliable scaffold. Finally, the scaffold serves as a strong prior for a novel view generator to produce photorealistic and geometrically accurate views at arbitrary cameras. By explicitly conditioning on a 3D-consistent scaffold to perform reconstruction, One2Scene works stably under large camera motions, supporting immersive scene exploration. Extensive experiments show that One2Scene substantially outperforms state-of-the-art methods in panorama depth estimation, feed-forward 360° reconstruction, and explorable 3D scene generation. Project page: https://one2scene5406.github.io/","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2602.19766.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"One2Scene: Geometric Consistent Explorable 3D Scene Generation from a Single Image"}],"highlightedSummary":[{"type":"highlight","text":"Generating explorable 3D scenes from a single image is a highly challenging problem in 3D vision."},{"type":"text","text":" Existing methods struggle to support free exploration, often producing severe geometric distortions and noisy artifacts when the viewpoint moves far from the original perspective. We introduce One2Scene, an effective framework that decomposes this ill-posed problem into three tractable sub-tasks to enable immersive explorable scene generation. We first use a panorama generator to produce anchor views from a single input image as initialization. Then, we lift these 2D anchors into an explicit 3D geometric scaffold via a generalizable, feed-forward Gaussian Splatting network. Instead of treating the panorama as a single image for reconstruction, we project it into multiple sparse anchor views and reformulate the reconstruction task as multi-view stereo matching, which allows us to leverage robust geometric priors learned from large-scale multi-view datasets. A bidirectional feature fusion module is used to enforce cross-view consistency, yielding an efficient and geometrically reliable scaffold. Finally, the scaffold serves as a strong prior for a novel view generator to produce photorealistic and geometrically accurate views at arbitrary cameras. By explicitly conditioning on a 3D-consistent scaffold to perform reconstruction, One2Scene works stably under large camera motions, supporting immersive scene exploration. Extensive experiments show that One2Scene substantially outperforms state-of-the-art methods in panorama depth estimation, feed-forward 360° reconstruction, and explorable 3D scene generation. Project page: https://one2scene5406.github.io/"}]},{"paper":{"id":"2607.05392","authors":[{"_id":"6a4d0dcc25849b193a834818","user":{"_id":"62262603a8218a9b862b18cc","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1648321672399-62262603a8218a9b862b18cc.jpeg","isPro":false,"fullname":"Paul Engstler","user":"paulengstler","type":"user","name":"paulengstler"},"name":"Paul Engstler","status":"claimed_verified","statusLastChangedAt":"2026-07-09T16:40:22.912Z","hidden":false},{"_id":"6a4d0dcc25849b193a834819","name":"Iro Laina","hidden":false},{"_id":"6a4d0dcc25849b193a83481a","name":"Christian Rupprecht","hidden":false},{"_id":"6a4d0dcc25849b193a83481b","name":"Andrea Vedaldi","hidden":false}],"publishedAt":"2026-07-06T00:00:00.000Z","title":"SynCity 3000: Bootstrapping Scene-Scale 3D Diffusion","summary":"We present SynCity 3000, a framework for generating 3D scenes that are globally coherent while enabling fine-grained layout control. Building on the ability of current image-to-3D generators to produce complex 3D assets from a single image, we extend this capability to the scale of entire scenes by adapting the generator to be applicable as a convolutional operator. We achieve this by fine-tuning the model on scene-like data generated by a new synthetic data engine, which we propose to address the scarcity of 3D scene data for training. The convolutional generator is then applied to a dimetric image of the entire scene, generated from the user prompt, resulting in 3D scenes of arbitrary size and complexity. Across diverse prompts and layouts, SynCity 3000 produces large, coherent, and detailed scenes, addressing the shortcomings of prior approaches to 3D scene generation.","upvotes":4,"discussionId":"6a4d0dcc25849b193a83481c","projectPage":"https://research.paulengstler.com/syncity-3k/","githubRepo":"https://github.com/paulengstler/syncity-3k","githubRepoAddedBy":"user","ai_summary":"SynCity 3000 generates large, coherent 3D scenes by adapting image-to-3D generators as convolutional operators through fine-tuning on synthetic scene data.","ai_keywords":["image-to-3D generators","convolutional operator","fine-tuning","synthetic data engine","dimetric image","3D scene generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":22,"organization":{"_id":"627bbc28fbab61b048eba8b6","name":"Oxford","fullname":"University of Oxford","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/68e396f2b5bb631e9b2fac9a/u0ey2LfYu6uG6iu8m_kH7.png"}},"publishedAt":"2026-07-05T20:00:00.000Z","title":"SynCity 3000: Bootstrapping Scene-Scale 3D Diffusion","summary":"We present SynCity 3000, a framework for generating 3D scenes that are globally coherent while enabling fine-grained layout control. Building on the ability of current image-to-3D generators to produce complex 3D assets from a single image, we extend this capability to the scale of entire scenes by adapting the generator to be applicable as a convolutional operator. We achieve this by fine-tuning the model on scene-like data generated by a new synthetic data engine, which we propose to address the scarcity of 3D scene data for training. The convolutional generator is then applied to a dimetric image of the entire scene, generated from the user prompt, resulting in 3D scenes of arbitrary size and complexity. Across diverse prompts and layouts, SynCity 3000 produces large, coherent, and detailed scenes, addressing the shortcomings of prior approaches to 3D scene generation.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2607.05392.png","numComments":2,"upvoted":false,"organization":{"_id":"627bbc28fbab61b048eba8b6","name":"Oxford","fullname":"University of Oxford","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/68e396f2b5bb631e9b2fac9a/u0ey2LfYu6uG6iu8m_kH7.png"},"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"SynCity 3000: Bootstrapping Scene-Scale 3D Diffusion"}],"highlightedSummary":[{"type":"text","text":"We present SynCity 3000, a framework for generating 3D scenes that are globally coherent while enabling fine-grained layout control. Building on the ability of current image-to-3D generators to produce complex 3D assets from a single image, we extend this capability to the scale of entire scenes by adapting the generator to be applicable as a convolutional operator. We achieve this by fine-tuning the model on scene-like data generated by a new synthetic data engine, which we propose to address the scarcity of 3D scene data for training. The convolutional generator is then applied to a dimetric image of the entire scene, generated from the user prompt, resulting in 3D scenes of arbitrary size and complexity. "},{"type":"highlight","text":"Across diverse prompts and layouts, SynCity 3000 produces large, coherent, and detailed scenes, addressing the shortcomings of prior approaches to 3D scene generation."}]},{"paper":{"id":"2512.13683","authors":[{"_id":"694137b865f1e24a1178076d","name":"Lu Ling","hidden":false},{"_id":"694137b865f1e24a1178076e","name":"Yunhao Ge","hidden":false},{"_id":"694137b865f1e24a1178076f","name":"Yichen Sheng","hidden":false},{"_id":"694137b865f1e24a11780770","name":"Aniket Bera","hidden":false}],"publishedAt":"2025-12-15T18:59:13.000Z","title":"I-Scene: 3D Instance Models are Implicit Generalizable Spatial Learners","summary":"Generalization remains the central challenge for interactive 3D scene generation. Existing learning-based approaches ground spatial understanding in limited scene dataset, restricting generalization to new layouts. We instead reprogram a pre-trained 3D instance generator to act as a scene level learner, replacing dataset-bounded supervision with model-centric spatial supervision. This reprogramming unlocks the generator transferable spatial knowledge, enabling generalization to unseen layouts and novel object compositions. Remarkably, spatial reasoning still emerges even when the training scenes are randomly composed objects. This demonstrates that the generator's transferable scene prior provides a rich learning signal for inferring proximity, support, and symmetry from purely geometric cues. Replacing widely used canonical space, we instantiate this insight with a view-centric formulation of the scene space, yielding a fully feed-forward, generalizable scene generator that learns spatial relations directly from the instance model. Quantitative and qualitative results show that a 3D instance generator is an implicit spatial learner and reasoner, pointing toward foundation models for interactive 3D scene understanding and generation. Project page: https://luling06.github.io/I-Scene-project/","upvotes":4,"discussionId":"694137b965f1e24a11780771","projectPage":"https://luling06.github.io/I-Scene-project/","githubRepo":"https://github.com/LuLing06/I-Scene-project","githubRepoAddedBy":"user","ai_summary":"A pre-trained 3D instance generator is reprogrammed to generalize spatial understanding in new layouts by learning directly from geometric cues, demonstrating its potential as a foundation model for interactive 3D scene generation.","ai_keywords":["3D instance generator","spatial understanding","dataset-bounded supervision","model-centric spatial supervision","transferable spatial knowledge","spatial reasoning","canonical space","view-centric formulation","scene space","feed-forward","generalizable scene generator","spatial relations","implicit spatial learner","interactive 3D scene understanding"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":115},"publishedAt":"2025-12-15T13:59:13.000Z","title":"I-Scene: 3D Instance Models are Implicit Generalizable Spatial Learners","summary":"Generalization remains the central challenge for interactive 3D scene generation. Existing learning-based approaches ground spatial understanding in limited scene dataset, restricting generalization to new layouts. We instead reprogram a pre-trained 3D instance generator to act as a scene level learner, replacing dataset-bounded supervision with model-centric spatial supervision. This reprogramming unlocks the generator transferable spatial knowledge, enabling generalization to unseen layouts and novel object compositions. Remarkably, spatial reasoning still emerges even when the training scenes are randomly composed objects. This demonstrates that the generator's transferable scene prior provides a rich learning signal for inferring proximity, support, and symmetry from purely geometric cues. Replacing widely used canonical space, we instantiate this insight with a view-centric formulation of the scene space, yielding a fully feed-forward, generalizable scene generator that learns spatial relations directly from the instance model. Quantitative and qualitative results show that a 3D instance generator is an implicit spatial learner and reasoner, pointing toward foundation models for interactive 3D scene understanding and generation. Project page: https://luling06.github.io/I-Scene-project/","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2512.13683.png","numComments":2,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"I-Scene: 3D Instance Models are Implicit Generalizable Spatial Learners"}],"highlightedSummary":[{"type":"text","text":"Generalization remains the central challenge for interactive 3D scene generation. Existing learning-based approaches ground spatial understanding in limited scene dataset, restricting generalization to new layouts. We instead reprogram a pre-trained 3D instance generator to act as a scene level learner, replacing dataset-bounded supervision with model-centric spatial supervision. This reprogramming unlocks the generator transferable spatial knowledge, enabling generalization to unseen layouts and novel object compositions. Remarkably, spatial reasoning still emerges even when the training scenes are randomly composed objects. This demonstrates that the generator's transferable scene prior provides a rich learning signal for inferring proximity, support, and symmetry from purely geometric cues. Replacing widely used canonical space, we instantiate this insight with a view-centric formulation of the scene space, yielding a fully feed-forward, generalizable scene generator that learns spatial relations directly from the instance model. Quantitative and qualitative results show that a 3D instance generator is an implicit spatial learner and reasoner, pointing toward foundation models for interactive 3D scene understanding and generation. Project page: https://luling06.github.io/I-Scene-project/"}]},{"paper":{"id":"2506.20703","authors":[{"_id":"685ef79a9e7509383d29a992","user":{"_id":"631219973a773344e06c2fd4","avatarUrl":"/avatars/44a09471f7b0d551505cb14cc878a09f.svg","isPro":false,"fullname":"Vaibhav Vavilala","user":"vv1233","type":"user","name":"vv1233"},"name":"Vaibhav Vavilala","status":"claimed_verified","statusLastChangedAt":"2025-06-27T20:30:31.925Z","hidden":false},{"_id":"685ef79a9e7509383d29a993","name":"Seemandhar Jain","hidden":false},{"_id":"685ef79a9e7509383d29a994","name":"Rahul Vasanth","hidden":false},{"_id":"685ef79a9e7509383d29a995","name":"D. A. Forsyth","hidden":false},{"_id":"685ef79a9e7509383d29a996","user":{"_id":"6303d5924ec2dfa82a5627de","avatarUrl":"/avatars/1e433b7fb9d30b9f42dec487d658b1e3.svg","isPro":false,"fullname":"Anand Bhattad","user":"anandbhattad","type":"user","name":"anandbhattad"},"name":"Anand Bhattad","status":"claimed_verified","statusLastChangedAt":"2025-06-29T10:14:14.520Z","hidden":false}],"publishedAt":"2025-06-25T17:59:55.000Z","title":"Generative Blocks World: Moving Things Around in Pictures","summary":"We describe Generative Blocks World to interact with the scene of a generated\nimage by manipulating simple geometric abstractions. Our method represents\nscenes as assemblies of convex 3D primitives, and the same scene can be\nrepresented by different numbers of primitives, allowing an editor to move\neither whole structures or small details. Once the scene geometry has been\nedited, the image is generated by a flow-based method which is conditioned on\ndepth and a texture hint. Our texture hint takes into account the modified 3D\nprimitives, exceeding texture-consistency provided by existing key-value\ncaching techniques. These texture hints (a) allow accurate object and camera\nmoves and (b) largely preserve the identity of objects depicted. Quantitative\nand qualitative experiments demonstrate that our approach outperforms prior\nworks in visual fidelity, editability, and compositional generalization.","upvotes":5,"discussionId":"685ef79a9e7509383d29a997","ai_summary":"A generative method that edits 3D scenes using convex primitives and regenerates images with enhanced texture consistency and visual fidelity.","ai_keywords":["flow-based method","convex 3D primitives","depth","texture hint","key-value caching","visual fidelity","editability","compositional generalization"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-06-25T13:59:55.000Z","title":"Generative Blocks World: Moving Things Around in Pictures","summary":"We describe Generative Blocks World to interact with the scene of a generated\nimage by manipulating simple geometric abstractions. Our method represents\nscenes as assemblies of convex 3D primitives, and the same scene can be\nrepresented by different numbers of primitives, allowing an editor to move\neither whole structures or small details. Once the scene geometry has been\nedited, the image is generated by a flow-based method which is conditioned on\ndepth and a texture hint. Our texture hint takes into account the modified 3D\nprimitives, exceeding texture-consistency provided by existing key-value\ncaching techniques. These texture hints (a) allow accurate object and camera\nmoves and (b) largely preserve the identity of objects depicted. Quantitative\nand qualitative experiments demonstrate that our approach outperforms prior\nworks in visual fidelity, editability, and compositional generalization.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2506.20703.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Generative Blocks World: Moving Things Around in Pictures"}],"highlightedSummary":[{"type":"text","text":"We describe Generative Blocks World to interact with the scene of a generated\nimage by manipulating simple geometric abstractions. "},{"type":"highlight","text":"Our method represents\nscenes as assemblies of convex 3D primitives, and the same scene can be\nrepresented by different numbers of primitives, allowing an editor to move\neither whole structures or small details."},{"type":"text","text":" Once the scene geometry has been\nedited, the image is generated by a flow-based method which is conditioned on\ndepth and a texture hint. Our texture hint takes into account the modified 3D\nprimitives, exceeding texture-consistency provided by existing key-value\ncaching techniques. These texture hints (a) allow accurate object and camera\nmoves and (b) largely preserve the identity of objects depicted. Quantitative\nand qualitative experiments demonstrate that our approach outperforms prior\nworks in visual fidelity, editability, and compositional generalization."}]},{"paper":{"id":"2512.09164","authors":[{"_id":"693a392174fced5bf9c32428","user":{"_id":"6753fe5ef49782992ebc00b6","avatarUrl":"/avatars/899954f5582237520ee9491ac85bc979.svg","isPro":false,"fullname":"Jin Cao","user":"TmaKiss","type":"user","name":"TmaKiss"},"name":"Jin Cao","status":"claimed_verified","statusLastChangedAt":"2026-07-31T08:45:05.129Z","hidden":false},{"_id":"693a392174fced5bf9c32429","name":"Hong-Xing Yu","hidden":false},{"_id":"693a392174fced5bf9c3242a","name":"Jiajun Wu","hidden":false}],"publishedAt":"2025-12-09T22:21:07.000Z","title":"WonderZoom: Multi-Scale 3D World Generation","summary":"We present WonderZoom, a novel approach to generating 3D scenes with contents across multiple spatial scales from a single image. Existing 3D world generation models remain limited to single-scale synthesis and cannot produce coherent scene contents at varying granularities. The fundamental challenge is the lack of a scale-aware 3D representation capable of generating and rendering content with largely different spatial sizes. WonderZoom addresses this through two key innovations: (1) scale-adaptive Gaussian surfels for generating and real-time rendering of multi-scale 3D scenes, and (2) a progressive detail synthesizer that iteratively generates finer-scale 3D contents. Our approach enables users to \"zoom into\" a 3D region and auto-regressively synthesize previously non-existent fine details from landscapes to microscopic features. Experiments demonstrate that WonderZoom significantly outperforms state-of-the-art video and 3D models in both quality and alignment, enabling multi-scale 3D world creation from a single image. We show video results and an interactive viewer of generated multi-scale 3D worlds in https://wonderzoom.github.io/","upvotes":13,"discussionId":"693a392174fced5bf9c3242b","projectPage":"https://wonderzoom.github.io/","githubRepo":"https://github.com/jin-cao-tma/WonderZoom","githubRepoAddedBy":"auto","ai_summary":"WonderZoom generates multi-scale 3D scenes from a single image using scale-adaptive Gaussian surfels and a progressive detail synthesizer, outperforming existing models in quality and alignment.","ai_keywords":["scale-adaptive Gaussian surfels","progressive detail synthesizer","multi-scale 3D scenes","single image","3D world generation","real-time rendering","fine details","video results","interactive viewer"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":207},"publishedAt":"2025-12-09T17:21:07.000Z","title":"WonderZoom: Multi-Scale 3D World Generation","summary":"We present WonderZoom, a novel approach to generating 3D scenes with contents across multiple spatial scales from a single image. Existing 3D world generation models remain limited to single-scale synthesis and cannot produce coherent scene contents at varying granularities. The fundamental challenge is the lack of a scale-aware 3D representation capable of generating and rendering content with largely different spatial sizes. WonderZoom addresses this through two key innovations: (1) scale-adaptive Gaussian surfels for generating and real-time rendering of multi-scale 3D scenes, and (2) a progressive detail synthesizer that iteratively generates finer-scale 3D contents. Our approach enables users to \"zoom into\" a 3D region and auto-regressively synthesize previously non-existent fine details from landscapes to microscopic features. Experiments demonstrate that WonderZoom significantly outperforms state-of-the-art video and 3D models in both quality and alignment, enabling multi-scale 3D world creation from a single image. We show video results and an interactive viewer of generated multi-scale 3D worlds in https://wonderzoom.github.io/","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2512.09164.png","numComments":2,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"WonderZoom: Multi-Scale 3D World Generation"}],"highlightedSummary":[{"type":"highlight","text":"We present WonderZoom, a novel approach to generating 3D scenes with contents across multiple spatial scales from a single image."},{"type":"text","text":" Existing 3D world generation models remain limited to single-scale synthesis and cannot produce coherent scene contents at varying granularities. The fundamental challenge is the lack of a scale-aware 3D representation capable of generating and rendering content with largely different spatial sizes. WonderZoom addresses this through two key innovations: (1) scale-adaptive Gaussian surfels for generating and real-time rendering of multi-scale 3D scenes, and (2) a progressive detail synthesizer that iteratively generates finer-scale 3D contents. Our approach enables users to \"zoom into\" a 3D region and auto-regressively synthesize previously non-existent fine details from landscapes to microscopic features. Experiments demonstrate that WonderZoom significantly outperforms state-of-the-art video and 3D models in both quality and alignment, enabling multi-scale 3D world creation from a single image. We show video results and an interactive viewer of generated multi-scale 3D worlds in https://wonderzoom.github.io/"}]},{"paper":{"id":"2502.14247","authors":[{"_id":"67be3ebc1c80786468704721","name":"Jiayu Yang","hidden":false},{"_id":"67be3ebc1c80786468704722","name":"Taizhang Shang","hidden":false},{"_id":"67be3ebc1c80786468704723","name":"Weixuan Sun","hidden":false},{"_id":"67be3ebc1c80786468704724","name":"Xibin Song","hidden":false},{"_id":"67be3ebc1c80786468704725","name":"Ziang Cheng","hidden":false},{"_id":"67be3ebc1c80786468704726","name":"Senbo Wang","hidden":false},{"_id":"67be3ebc1c80786468704727","name":"Shenzhou Chen","hidden":false},{"_id":"67be3ebc1c80786468704728","name":"Weizhe Liu","hidden":false},{"_id":"67be3ebc1c80786468704729","name":"Hongdong Li","hidden":false},{"_id":"67be3ebc1c8078646870472a","name":"Pan Ji","hidden":false}],"publishedAt":"2025-02-20T04:22:30.000Z","title":"Pandora3D: A Comprehensive Framework for High-Quality 3D Shape and\n  Texture Generation","summary":"This report presents a comprehensive framework for generating high-quality 3D\nshapes and textures from diverse input prompts, including single images,\nmulti-view images, and text descriptions. The framework consists of 3D shape\ngeneration and texture generation. (1). The 3D shape generation pipeline\nemploys a Variational Autoencoder (VAE) to encode implicit 3D geometries into a\nlatent space and a diffusion network to generate latents conditioned on input\nprompts, with modifications to enhance model capacity. An alternative\nArtist-Created Mesh (AM) generation approach is also explored, yielding\npromising results for simpler geometries. (2). Texture generation involves a\nmulti-stage process starting with frontal images generation followed by\nmulti-view images generation, RGB-to-PBR texture conversion, and\nhigh-resolution multi-view texture refinement. A consistency scheduler is\nplugged into every stage, to enforce pixel-wise consistency among multi-view\ntextures during inference, ensuring seamless integration.\n  The pipeline demonstrates effective handling of diverse input formats,\nleveraging advanced neural architectures and novel methodologies to produce\nhigh-quality 3D content. This report details the system architecture,\nexperimental results, and potential future directions to improve and expand the\nframework. The source code and pretrained weights are released at:\nhttps://github.com/Tencent/Tencent-XR-3DGen.","upvotes":6,"discussionId":"67be3ec21c80786468704886","githubRepo":"https://github.com/tencent/tencent-xr-3dgen","githubRepoAddedBy":"auto","ai_summary":"The framework generates high-quality 3D shapes and textures from various inputs using VAE, diffusion networks, and a multi-stage texture generation process with consistency enforcement.","ai_keywords":["Variational Autoencoder","VAE","diffusion network","Artist-Created Mesh","AM","RGB-to-PBR","consistency scheduler"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":357},"publishedAt":"2025-02-19T23:22:30.000Z","title":"Pandora3D: A Comprehensive Framework for High-Quality 3D Shape and\n  Texture Generation","summary":"This report presents a comprehensive framework for generating high-quality 3D\nshapes and textures from diverse input prompts, including single images,\nmulti-view images, and text descriptions. The framework consists of 3D shape\ngeneration and texture generation. (1). The 3D shape generation pipeline\nemploys a Variational Autoencoder (VAE) to encode implicit 3D geometries into a\nlatent space and a diffusion network to generate latents conditioned on input\nprompts, with modifications to enhance model capacity. An alternative\nArtist-Created Mesh (AM) generation approach is also explored, yielding\npromising results for simpler geometries. (2). Texture generation involves a\nmulti-stage process starting with frontal images generation followed by\nmulti-view images generation, RGB-to-PBR texture conversion, and\nhigh-resolution multi-view texture refinement. A consistency scheduler is\nplugged into every stage, to enforce pixel-wise consistency among multi-view\ntextures during inference, ensuring seamless integration.\n  The pipeline demonstrates effective handling of diverse input formats,\nleveraging advanced neural architectures and novel methodologies to produce\nhigh-quality 3D content. This report details the system architecture,\nexperimental results, and potential future directions to improve and expand the\nframework. The source code and pretrained weights are released at:\nhttps://github.com/Tencent/Tencent-XR-3DGen.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2502.14247.png","numComments":2,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Pandora3D: A Comprehensive Framework for High-Quality 3D Shape and\n  Texture Generation"}],"highlightedSummary":[{"type":"highlight","text":"This report presents a comprehensive framework for generating high-quality 3D\nshapes and textures from diverse input prompts, including single images,\nmulti-view images, and text descriptions."},{"type":"text","text":" The framework consists of 3D shape\ngeneration and texture generation. (1). The 3D shape generation pipeline\nemploys a Variational Autoencoder (VAE) to encode implicit 3D geometries into a\nlatent space and a diffusion network to generate latents conditioned on input\nprompts, with modifications to enhance model capacity. An alternative\nArtist-Created Mesh (AM) generation approach is also explored, yielding\npromising results for simpler geometries. (2). Texture generation involves a\nmulti-stage process starting with frontal images generation followed by\nmulti-view images generation, RGB-to-PBR texture conversion, and\nhigh-resolution multi-view texture refinement. A consistency scheduler is\nplugged into every stage, to enforce pixel-wise consistency among multi-view\ntextures during inference, ensuring seamless integration.\n  The pipeline demonstrates effective handling of diverse input formats,\nleveraging advanced neural architectures and novel methodologies to produce\nhigh-quality 3D content. This report details the system architecture,\nexperimental results, and potential future directions to improve and expand the\nframework. The source code and pretrained weights are released at:\nhttps://github.com/Tencent/Tencent-XR-3DGen."}]},{"paper":{"id":"2604.11331","authors":[{"_id":"69f448e5ab6004e666b0083c","name":"Dongxu Wei","hidden":false},{"_id":"69f448e5ab6004e666b0083d","user":{"_id":"6580440ae77395a0c8399477","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6580440ae77395a0c8399477/hTNLwyfpHDTQ_fDAR6d7k.jpeg","isPro":false,"fullname":"XuQi","user":"insomnia7","type":"user","name":"insomnia7"},"name":"Qi Xu","status":"claimed_verified","statusLastChangedAt":"2026-05-04T10:14:04.387Z","hidden":false},{"_id":"69f448e5ab6004e666b0083e","name":"Zhiqi Li","hidden":false},{"_id":"69f448e5ab6004e666b0083f","name":"Hangning Zhou","hidden":false},{"_id":"69f448e5ab6004e666b00840","name":"Cong Qiu","hidden":false},{"_id":"69f448e5ab6004e666b00841","name":"Hailong Qin","hidden":false},{"_id":"69f448e5ab6004e666b00842","name":"Mu Yang","hidden":false},{"_id":"69f448e5ab6004e666b00843","name":"Zhaopeng Cui","hidden":false},{"_id":"69f448e5ab6004e666b00844","name":"Peidong Liu","hidden":false}],"publishedAt":"2026-04-13T00:00:00.000Z","title":"Any 3D Scene is Worth 1K Tokens: 3D-Grounded Representation for Scene Generation at Scale","summary":"3D scene generation has long been dominated by 2D multi-view or video diffusion models. This is due not only to the lack of scene-level 3D latent representation, but also to the fact that most scene-level 3D visual data exists in the form of multi-view images or videos, which are naturally compatible with 2D diffusion architectures. Typically, these 2D-based approaches degrade 3D spatial extrapolation to 2D temporal extension, which introduces two fundamental issues: (i) representing 3D scenes via 2D views leads to significant representation redundancy, and (ii) latent space rooted in 2D inherently limits the spatial consistency of the generated 3D scenes. In this paper, we propose, for the first time, to perform 3D scene generation directly within an implicit 3D latent space to address these limitations. First, we repurpose frozen 2D representation encoders to construct our 3D Representation Autoencoder (3DRAE), which grounds view-coupled 2D semantic representations into a view-decoupled 3D latent representation. This enables representing 3D scenes observed from arbitrary numbers of views--at any resolution and aspect ratio--with fixed complexity and rich semantics. Then we introduce 3D Diffusion Transformer (3DDiT), which performs diffusion modeling in this 3D latent space, achieving remarkably efficient and spatially consistent 3D scene generation while supporting diverse conditioning configurations. Moreover, since our approach directly generates a 3D scene representation, it can be decoded to images and optional point maps along arbitrary camera trajectories without requiring per-trajectory diffusion sampling pass, which is common in 2D-based approaches.","upvotes":0,"discussionId":"69f448e5ab6004e666b00845","ai_summary":"3D scene generation is improved by using implicit 3D latent space through a 3D Representation Autoencoder and 3D Diffusion Transformer, enabling efficient and consistent 3D scene representation and generation.","ai_keywords":["3D scene generation","2D multi-view","video diffusion models","3D latent representation","3D Representation Autoencoder","3D Diffusion Transformer","view-coupled","view-decoupled","implicit 3D latent space","diffusion modeling","spatial consistency","conditioning configurations","camera trajectories"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2026-04-12T20:00:00.000Z","title":"Any 3D Scene is Worth 1K Tokens: 3D-Grounded Representation for Scene Generation at Scale","summary":"3D scene generation has long been dominated by 2D multi-view or video diffusion models. This is due not only to the lack of scene-level 3D latent representation, but also to the fact that most scene-level 3D visual data exists in the form of multi-view images or videos, which are naturally compatible with 2D diffusion architectures. Typically, these 2D-based approaches degrade 3D spatial extrapolation to 2D temporal extension, which introduces two fundamental issues: (i) representing 3D scenes via 2D views leads to significant representation redundancy, and (ii) latent space rooted in 2D inherently limits the spatial consistency of the generated 3D scenes. In this paper, we propose, for the first time, to perform 3D scene generation directly within an implicit 3D latent space to address these limitations. First, we repurpose frozen 2D representation encoders to construct our 3D Representation Autoencoder (3DRAE), which grounds view-coupled 2D semantic representations into a view-decoupled 3D latent representation. This enables representing 3D scenes observed from arbitrary numbers of views--at any resolution and aspect ratio--with fixed complexity and rich semantics. Then we introduce 3D Diffusion Transformer (3DDiT), which performs diffusion modeling in this 3D latent space, achieving remarkably efficient and spatially consistent 3D scene generation while supporting diverse conditioning configurations. Moreover, since our approach directly generates a 3D scene representation, it can be decoded to images and optional point maps along arbitrary camera trajectories without requiring per-trajectory diffusion sampling pass, which is common in 2D-based approaches.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2604.11331.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Any 3D Scene is Worth 1K Tokens: 3D-Grounded Representation for Scene Generation at Scale"}],"highlightedSummary":[{"type":"text","text":"3D scene generation has long been dominated by 2D multi-view or video diffusion models. This is due not only to the lack of scene-level 3D latent representation, but also to the fact that most scene-level 3D visual data exists in the form of multi-view images or videos, which are naturally compatible with 2D diffusion architectures. Typically, these 2D-based approaches degrade 3D spatial extrapolation to 2D temporal extension, which introduces two fundamental issues: (i) representing 3D scenes via 2D views leads to significant representation redundancy, and (ii) latent space rooted in 2D inherently limits the spatial consistency of the generated 3D scenes. "},{"type":"highlight","text":"In this paper, we propose, for the first time, to perform 3D scene generation directly within an implicit 3D latent space to address these limitations."},{"type":"text","text":" First, we repurpose frozen 2D representation encoders to construct our 3D Representation Autoencoder (3DRAE), which grounds view-coupled 2D semantic representations into a view-decoupled 3D latent representation. This enables representing 3D scenes observed from arbitrary numbers of views--at any resolution and aspect ratio--with fixed complexity and rich semantics. Then we introduce 3D Diffusion Transformer (3DDiT), which performs diffusion modeling in this 3D latent space, achieving remarkably efficient and spatially consistent 3D scene generation while supporting diverse conditioning configurations. Moreover, since our approach directly generates a 3D scene representation, it can be decoded to images and optional point maps along arbitrary camera trajectories without requiring per-trajectory diffusion sampling pass, which is common in 2D-based approaches."}]},{"paper":{"id":"2506.20601","authors":[{"_id":"688b79948c434640078cc29b","name":"Rui Huang","hidden":false},{"_id":"688b79948c434640078cc29c","name":"Guangyao Zhai","hidden":false},{"_id":"688b79948c434640078cc29d","name":"Zuria Bauer","hidden":false},{"_id":"688b79948c434640078cc29e","name":"Marc Pollefeys","hidden":false},{"_id":"688b79948c434640078cc29f","name":"Federico Tombari","hidden":false},{"_id":"688b79948c434640078cc2a0","name":"Leonidas Guibas","hidden":false},{"_id":"688b79948c434640078cc2a1","name":"Gao Huang","hidden":false},{"_id":"688b79948c434640078cc2a2","name":"Francis Engelmann","hidden":false}],"publishedAt":"2025-06-25T16:40:17.000Z","title":"Video Perception Models for 3D Scene Synthesis","summary":"Traditionally, 3D scene synthesis requires expert knowledge and significant\nmanual effort. Automating this process could greatly benefit fields such as\narchitectural design, robotics simulation, virtual reality, and gaming. Recent\napproaches to 3D scene synthesis often rely on the commonsense reasoning of\nlarge language models (LLMs) or strong visual priors of modern image generation\nmodels. However, current LLMs demonstrate limited 3D spatial reasoning ability,\nwhich restricts their ability to generate realistic and coherent 3D scenes.\nMeanwhile, image generation-based methods often suffer from constraints in\nviewpoint selection and multi-view inconsistencies. In this work, we present\nVideo Perception models for 3D Scene synthesis (VIPScene), a novel framework\nthat exploits the encoded commonsense knowledge of the 3D physical world in\nvideo generation models to ensure coherent scene layouts and consistent object\nplacements across views. VIPScene accepts both text and image prompts and\nseamlessly integrates video generation, feedforward 3D reconstruction, and\nopen-vocabulary perception models to semantically and geometrically analyze\neach object in a scene. This enables flexible scene synthesis with high realism\nand structural consistency. For more precise analysis, we further introduce\nFirst-Person View Score (FPVScore) for coherence and plausibility evaluation,\nutilizing continuous first-person perspective to capitalize on the reasoning\nability of multimodal large language models. Extensive experiments show that\nVIPScene significantly outperforms existing methods and generalizes well across\ndiverse scenarios. The code will be released.","upvotes":1,"discussionId":"688b79948c434640078cc2a3","ai_summary":"VIPScene integrates video generation, 3D reconstruction, and perception models to synthesize realistic and consistent 3D scenes from text and image prompts, outperforming existing methods.","ai_keywords":["Video Perception models","3D Scene synthesis","VIPScene","commonsense knowledge","video generation models","feedforward 3D reconstruction","open-vocabulary perception models","First-Person View Score","FPVScore","multimodal large language models"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-06-25T12:40:17.000Z","title":"Video Perception Models for 3D Scene Synthesis","summary":"Traditionally, 3D scene synthesis requires expert knowledge and significant\nmanual effort. Automating this process could greatly benefit fields such as\narchitectural design, robotics simulation, virtual reality, and gaming. Recent\napproaches to 3D scene synthesis often rely on the commonsense reasoning of\nlarge language models (LLMs) or strong visual priors of modern image generation\nmodels. However, current LLMs demonstrate limited 3D spatial reasoning ability,\nwhich restricts their ability to generate realistic and coherent 3D scenes.\nMeanwhile, image generation-based methods often suffer from constraints in\nviewpoint selection and multi-view inconsistencies. In this work, we present\nVideo Perception models for 3D Scene synthesis (VIPScene), a novel framework\nthat exploits the encoded commonsense knowledge of the 3D physical world in\nvideo generation models to ensure coherent scene layouts and consistent object\nplacements across views. VIPScene accepts both text and image prompts and\nseamlessly integrates video generation, feedforward 3D reconstruction, and\nopen-vocabulary perception models to semantically and geometrically analyze\neach object in a scene. This enables flexible scene synthesis with high realism\nand structural consistency. For more precise analysis, we further introduce\nFirst-Person View Score (FPVScore) for coherence and plausibility evaluation,\nutilizing continuous first-person perspective to capitalize on the reasoning\nability of multimodal large language models. Extensive experiments show that\nVIPScene significantly outperforms existing methods and generalizes well across\ndiverse scenarios. The code will be released.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2506.20601.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Video Perception Models for 3D Scene Synthesis"}],"highlightedSummary":[{"type":"text","text":"Traditionally, 3D scene synthesis requires expert knowledge and significant\nmanual effort. Automating this process could greatly benefit fields such as\narchitectural design, robotics simulation, virtual reality, and gaming. Recent\napproaches to 3D scene synthesis often rely on the commonsense reasoning of\nlarge language models (LLMs) or strong visual priors of modern image generation\nmodels. However, current LLMs demonstrate limited 3D spatial reasoning ability,\nwhich restricts their ability to generate realistic and coherent 3D scenes.\n"},{"type":"highlight","text":"Meanwhile, image generation-based methods often suffer from constraints in\nviewpoint selection and multi-view inconsistencies."},{"type":"text","text":" In this work, we present\nVideo Perception models for 3D Scene synthesis (VIPScene), a novel framework\nthat exploits the encoded commonsense knowledge of the 3D physical world in\nvideo generation models to ensure coherent scene layouts and consistent object\nplacements across views. VIPScene accepts both text and image prompts and\nseamlessly integrates video generation, feedforward 3D reconstruction, and\nopen-vocabulary perception models to semantically and geometrically analyze\neach object in a scene. This enables flexible scene synthesis with high realism\nand structural consistency. For more precise analysis, we further introduce\nFirst-Person View Score (FPVScore) for coherence and plausibility evaluation,\nutilizing continuous first-person perspective to capitalize on the reasoning\nability of multimodal large language models. Extensive experiments show that\nVIPScene significantly outperforms existing methods and generalizes well across\ndiverse scenarios. The code will be released."}]},{"paper":{"id":"2510.15564","authors":[{"_id":"68f593d08589920bf4d320de","name":"Xiaoming Zhu","hidden":false},{"_id":"68f593d08589920bf4d320df","name":"Xu Huang","hidden":false},{"_id":"68f593d08589920bf4d320e0","name":"Qinghongbing Xie","hidden":false},{"_id":"68f593d08589920bf4d320e1","name":"Zhi Deng","hidden":false},{"_id":"68f593d08589920bf4d320e2","name":"Junsheng Yu","hidden":false},{"_id":"68f593d08589920bf4d320e3","name":"Yirui Guan","hidden":false},{"_id":"68f593d08589920bf4d320e4","name":"Zhongyuan Liu","hidden":false},{"_id":"68f593d08589920bf4d320e5","name":"Lin Zhu","hidden":false},{"_id":"68f593d08589920bf4d320e6","name":"Qijun Zhao","hidden":false},{"_id":"68f593d08589920bf4d320e7","name":"Ligang Liu","hidden":false},{"_id":"68f593d08589920bf4d320e8","name":"Long Zeng","hidden":false}],"publishedAt":"2025-10-17T11:48:08.000Z","title":"Imaginarium: Vision-guided High-Quality 3D Scene Layout Generation","summary":"Generating artistic and coherent 3D scene layouts is crucial in digital\ncontent creation. Traditional optimization-based methods are often constrained\nby cumbersome manual rules, while deep generative models face challenges in\nproducing content with richness and diversity. Furthermore, approaches that\nutilize large language models frequently lack robustness and fail to accurately\ncapture complex spatial relationships. To address these challenges, this paper\npresents a novel vision-guided 3D layout generation system. We first construct\na high-quality asset library containing 2,037 scene assets and 147 3D scene\nlayouts. Subsequently, we employ an image generation model to expand prompt\nrepresentations into images, fine-tuning it to align with our asset library. We\nthen develop a robust image parsing module to recover the 3D layout of scenes\nbased on visual semantics and geometric information. Finally, we optimize the\nscene layout using scene graphs and overall visual semantics to ensure logical\ncoherence and alignment with the images. Extensive user testing demonstrates\nthat our algorithm significantly outperforms existing methods in terms of\nlayout richness and quality. The code and dataset will be available at\nhttps://github.com/HiHiAllen/Imaginarium.","upvotes":11,"discussionId":"68f593d08589920bf4d320e9","githubRepo":"https://github.com/HiHiAllen/Imaginarium","githubRepoAddedBy":"auto","ai_summary":"A vision-guided 3D layout generation system uses an image generation model and scene graphs to produce rich and coherent 3D scenes from prompts.","ai_keywords":["image generation model","image parsing module","scene graphs","3D layout generation","visual semantics","geometric information","scene graphs","logical coherence"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":66},"publishedAt":"2025-10-17T07:48:08.000Z","title":"Imaginarium: Vision-guided High-Quality 3D Scene Layout Generation","summary":"Generating artistic and coherent 3D scene layouts is crucial in digital\ncontent creation. Traditional optimization-based methods are often constrained\nby cumbersome manual rules, while deep generative models face challenges in\nproducing content with richness and diversity. Furthermore, approaches that\nutilize large language models frequently lack robustness and fail to accurately\ncapture complex spatial relationships. To address these challenges, this paper\npresents a novel vision-guided 3D layout generation system. We first construct\na high-quality asset library containing 2,037 scene assets and 147 3D scene\nlayouts. Subsequently, we employ an image generation model to expand prompt\nrepresentations into images, fine-tuning it to align with our asset library. We\nthen develop a robust image parsing module to recover the 3D layout of scenes\nbased on visual semantics and geometric information. Finally, we optimize the\nscene layout using scene graphs and overall visual semantics to ensure logical\ncoherence and alignment with the images. Extensive user testing demonstrates\nthat our algorithm significantly outperforms existing methods in terms of\nlayout richness and quality. The code and dataset will be available at\nhttps://github.com/HiHiAllen/Imaginarium.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2510.15564.png","numComments":3,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Imaginarium: Vision-guided High-Quality 3D Scene Layout Generation"}],"highlightedSummary":[{"type":"text","text":"Generating artistic and coherent 3D scene layouts is crucial in digital\ncontent creation. Traditional optimization-based methods are often constrained\nby cumbersome manual rules, while deep generative models face challenges in\nproducing content with richness and diversity. Furthermore, approaches that\nutilize large language models frequently lack robustness and fail to accurately\ncapture complex spatial relationships. To address these challenges, this paper\npresents a novel vision-guided 3D layout generation system. We first construct\na high-quality asset library containing 2,037 scene assets and 147 3D scene\nlayouts. "},{"type":"highlight","text":"Subsequently, we employ an image generation model to expand prompt\nrepresentations into images, fine-tuning it to align with our asset library."},{"type":"text","text":" We\nthen develop a robust image parsing module to recover the 3D layout of scenes\nbased on visual semantics and geometric information. Finally, we optimize the\nscene layout using scene graphs and overall visual semantics to ensure logical\ncoherence and alignment with the images. Extensive user testing demonstrates\nthat our algorithm significantly outperforms existing methods in terms of\nlayout richness and quality. The code and dataset will be available at\nhttps://github.com/HiHiAllen/Imaginarium."}]},{"paper":{"id":"2412.04471","authors":[{"_id":"6776c4dc7e9c57be7229ddf6","name":"Vinayak Gupta","hidden":false},{"_id":"6776c4dc7e9c57be7229ddf7","user":{"_id":"654c09356a49f6f6e0034755","avatarUrl":"/avatars/dd1759e8a60f17c1ad33aafd35a7b672.svg","isPro":false,"fullname":"Yunze Man","user":"yunzeman","type":"user","name":"yunzeman"},"name":"Yunze Man","status":"claimed_verified","statusLastChangedAt":"2026-05-28T15:39:22.282Z","hidden":false},{"_id":"6776c4dc7e9c57be7229ddf8","name":"Yu-Xiong Wang","hidden":false}],"publishedAt":"2024-12-05T18:59:57.000Z","title":"PaintScene4D: Consistent 4D Scene Generation from Text Prompts","summary":"Recent advances in diffusion models have revolutionized 2D and 3D content\ncreation, yet generating photorealistic dynamic 4D scenes remains a significant\nchallenge. Existing dynamic 4D generation methods typically rely on distilling\nknowledge from pre-trained 3D generative models, often fine-tuned on synthetic\nobject datasets. Consequently, the resulting scenes tend to be object-centric\nand lack photorealism. While text-to-video models can generate more realistic\nscenes with motion, they often struggle with spatial understanding and provide\nlimited control over camera viewpoints during rendering. To address these\nlimitations, we present PaintScene4D, a novel text-to-4D scene generation\nframework that departs from conventional multi-view generative models in favor\nof a streamlined architecture that harnesses video generative models trained on\ndiverse real-world datasets. Our method first generates a reference video using\na video generation model, and then employs a strategic camera array selection\nfor rendering. We apply a progressive warping and inpainting technique to\nensure both spatial and temporal consistency across multiple viewpoints.\nFinally, we optimize multi-view images using a dynamic renderer, enabling\nflexible camera control based on user preferences. Adopting a training-free\narchitecture, our PaintScene4D efficiently produces realistic 4D scenes that\ncan be viewed from arbitrary trajectories. The code will be made publicly\navailable. Our project page is at https://paintscene4d.github.io/","upvotes":0,"discussionId":"6776c4de7e9c57be7229de8d","ai_summary":"PaintScene4D generates photorealistic 4D scenes using a video generation model and dynamic rendering techniques, offering flexibility in camera control and consistency across viewpoints.","ai_keywords":["diffusion models","4D scenes","video generation models","real-world datasets","strategic camera array selection","progressive warping","inpainting","temporal consistency","dynamic renderer","camera control"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-12-05T13:59:57.000Z","title":"PaintScene4D: Consistent 4D Scene Generation from Text Prompts","summary":"Recent advances in diffusion models have revolutionized 2D and 3D content\ncreation, yet generating photorealistic dynamic 4D scenes remains a significant\nchallenge. Existing dynamic 4D generation methods typically rely on distilling\nknowledge from pre-trained 3D generative models, often fine-tuned on synthetic\nobject datasets. Consequently, the resulting scenes tend to be object-centric\nand lack photorealism. While text-to-video models can generate more realistic\nscenes with motion, they often struggle with spatial understanding and provide\nlimited control over camera viewpoints during rendering. To address these\nlimitations, we present PaintScene4D, a novel text-to-4D scene generation\nframework that departs from conventional multi-view generative models in favor\nof a streamlined architecture that harnesses video generative models trained on\ndiverse real-world datasets. Our method first generates a reference video using\na video generation model, and then employs a strategic camera array selection\nfor rendering. We apply a progressive warping and inpainting technique to\nensure both spatial and temporal consistency across multiple viewpoints.\nFinally, we optimize multi-view images using a dynamic renderer, enabling\nflexible camera control based on user preferences. Adopting a training-free\narchitecture, our PaintScene4D efficiently produces realistic 4D scenes that\ncan be viewed from arbitrary trajectories. The code will be made publicly\navailable. Our project page is at https://paintscene4d.github.io/","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2412.04471.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"PaintScene4D: Consistent 4D Scene Generation from Text Prompts"}],"highlightedSummary":[{"type":"text","text":"Recent advances in diffusion models have revolutionized 2D and 3D content\ncreation, yet generating photorealistic dynamic 4D scenes remains a significant\nchallenge. Existing dynamic 4D generation methods typically rely on distilling\nknowledge from pre-trained 3D generative models, often fine-tuned on synthetic\nobject datasets. Consequently, the resulting scenes tend to be object-centric\nand lack photorealism. While text-to-video models can generate more realistic\nscenes with motion, they often struggle with spatial understanding and provide\nlimited control over camera viewpoints during rendering. "},{"type":"highlight","text":"To address these\nlimitations, we present PaintScene4D, a novel text-to-4D scene generation\nframework that departs from conventional multi-view generative models in favor\nof a streamlined architecture that harnesses video generative models trained on\ndiverse real-world datasets."},{"type":"text","text":" Our method first generates a reference video using\na video generation model, and then employs a strategic camera array selection\nfor rendering. We apply a progressive warping and inpainting technique to\nensure both spatial and temporal consistency across multiple viewpoints.\nFinally, we optimize multi-view images using a dynamic renderer, enabling\nflexible camera control based on user preferences. Adopting a training-free\narchitecture, our PaintScene4D efficiently produces realistic 4D scenes that\ncan be viewed from arbitrary trajectories. The code will be made publicly\navailable. Our project page is at https://paintscene4d.github.io/"}]},{"paper":{"id":"2601.04090","authors":[{"_id":"695fa6f82450f142afb3c59b","user":{"_id":"63e367d3fae035bdc4c347fc","avatarUrl":"/avatars/239da0c9287bf965348aff4890e94722.svg","isPro":false,"fullname":"Jiaxin Huang","user":"JaceyH919","type":"user","name":"JaceyH919"},"name":"Jiaxin Huang","status":"claimed_verified","statusLastChangedAt":"2026-01-09T08:35:46.425Z","hidden":false},{"_id":"695fa6f82450f142afb3c59c","name":"Yuanbo Yang","hidden":false},{"_id":"695fa6f82450f142afb3c59d","name":"Bangbang Yang","hidden":false},{"_id":"695fa6f82450f142afb3c59e","name":"Lin Ma","hidden":false},{"_id":"695fa6f82450f142afb3c59f","name":"Yuewen Ma","hidden":false},{"_id":"695fa6f82450f142afb3c5a0","name":"Yiyi Liao","hidden":false}],"publishedAt":"2026-01-07T16:57:30.000Z","title":"Gen3R: 3D Scene Generation Meets Feed-Forward Reconstruction","summary":"We present Gen3R, a method that bridges the strong priors of foundational reconstruction models and video diffusion models for scene-level 3D generation. We repurpose the VGGT reconstruction model to produce geometric latents by training an adapter on its tokens, which are regularized to align with the appearance latents of pre-trained video diffusion models. By jointly generating these disentangled yet aligned latents, Gen3R produces both RGB videos and corresponding 3D geometry, including camera poses, depth maps, and global point clouds. Experiments demonstrate that our approach achieves state-of-the-art results in single- and multi-image conditioned 3D scene generation. Additionally, our method can enhance the robustness of reconstruction by leveraging generative priors, demonstrating the mutual benefit of tightly coupling reconstruction and generative models.","upvotes":2,"discussionId":"695fa6f82450f142afb3c5a3","projectPage":"https://xdimlab.github.io/Gen3R/","githubRepo":"https://github.com/JaceyHuang/Gen3R","githubRepoAddedBy":"user","ai_summary":"Gen3R combines foundational reconstruction models with video diffusion models to generate 3D scenes with RGB videos and geometric information through aligned latents.","ai_keywords":["VGGT reconstruction model","video diffusion models","geometric latents","appearance latents","adapter","disentangled latents","camera poses","depth maps","global point clouds"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":384},"publishedAt":"2026-01-07T11:57:30.000Z","title":"Gen3R: 3D Scene Generation Meets Feed-Forward Reconstruction","summary":"We present Gen3R, a method that bridges the strong priors of foundational reconstruction models and video diffusion models for scene-level 3D generation. We repurpose the VGGT reconstruction model to produce geometric latents by training an adapter on its tokens, which are regularized to align with the appearance latents of pre-trained video diffusion models. By jointly generating these disentangled yet aligned latents, Gen3R produces both RGB videos and corresponding 3D geometry, including camera poses, depth maps, and global point clouds. Experiments demonstrate that our approach achieves state-of-the-art results in single- and multi-image conditioned 3D scene generation. Additionally, our method can enhance the robustness of reconstruction by leveraging generative priors, demonstrating the mutual benefit of tightly coupling reconstruction and generative models.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2601.04090.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Gen3R: 3D Scene Generation Meets Feed-Forward Reconstruction"}],"highlightedSummary":[{"type":"text","text":"We present Gen3R, a method that bridges the strong priors of foundational reconstruction models and video diffusion models for scene-level 3D generation. We repurpose the VGGT reconstruction model to produce geometric latents by training an adapter on its tokens, which are regularized to align with the appearance latents of pre-trained video diffusion models. By jointly generating these disentangled yet aligned latents, Gen3R produces both RGB videos and corresponding 3D geometry, including camera poses, depth maps, and global point clouds. "},{"type":"highlight","text":"Experiments demonstrate that our approach achieves state-of-the-art results in single- and multi-image conditioned 3D scene generation."},{"type":"text","text":" Additionally, our method can enhance the robustness of reconstruction by leveraging generative priors, demonstrating the mutual benefit of tightly coupling reconstruction and generative models."}]},{"paper":{"id":"2202.07135","authors":[{"_id":"64f7fbf79433a36edf47ede6","name":"Jinyang Yuan","hidden":false},{"_id":"64f7fbf79433a36edf47ede7","name":"Tonglin Chen","hidden":false},{"_id":"64f7fbf79433a36edf47ede8","name":"Bin Li","hidden":false},{"_id":"64f7fbf79433a36edf47ede9","name":"Xiangyang Xue","hidden":false}],"publishedAt":"2022-02-15T02:14:05.000Z","title":"Compositional Scene Representation Learning via Reconstruction: A Survey","summary":"Visual scenes are composed of visual concepts and have the property of\ncombinatorial explosion. An important reason for humans to efficiently learn\nfrom diverse visual scenes is the ability of compositional perception, and it\nis desirable for artificial intelligence to have similar abilities.\nCompositional scene representation learning is a task that enables such\nabilities. In recent years, various methods have been proposed to apply deep\nneural networks, which have been proven to be advantageous in representation\nlearning, to learn compositional scene representations via reconstruction,\nadvancing this research direction into the deep learning era. Learning via\nreconstruction is advantageous because it may utilize massive unlabeled data\nand avoid costly and laborious data annotation. In this survey, we first\noutline the current progress on reconstruction-based compositional scene\nrepresentation learning with deep neural networks, including development\nhistory and categorizations of existing methods from the perspectives of the\nmodeling of visual scenes and the inference of scene representations; then\nprovide benchmarks, including an open source toolbox to reproduce the benchmark\nexperiments, of representative methods that consider the most extensively\nstudied problem setting and form the foundation for other methods; and finally\ndiscuss the limitations of existing methods and future directions of this\nresearch topic.","upvotes":0,"discussionId":"64f7fbf99433a36edf47ee0c","ai_summary":"A survey on using deep neural networks for reconstruction-based compositional scene representation learning, covering methodological advancements, benchmarks, and future research directions.","ai_keywords":["deep neural networks","compositional scene representation learning","reconstruction","visual scenes","scene representations","open source toolbox"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2022-02-14T21:14:05.000Z","title":"Compositional Scene Representation Learning via Reconstruction: A Survey","summary":"Visual scenes are composed of visual concepts and have the property of\ncombinatorial explosion. An important reason for humans to efficiently learn\nfrom diverse visual scenes is the ability of compositional perception, and it\nis desirable for artificial intelligence to have similar abilities.\nCompositional scene representation learning is a task that enables such\nabilities. In recent years, various methods have been proposed to apply deep\nneural networks, which have been proven to be advantageous in representation\nlearning, to learn compositional scene representations via reconstruction,\nadvancing this research direction into the deep learning era. Learning via\nreconstruction is advantageous because it may utilize massive unlabeled data\nand avoid costly and laborious data annotation. In this survey, we first\noutline the current progress on reconstruction-based compositional scene\nrepresentation learning with deep neural networks, including development\nhistory and categorizations of existing methods from the perspectives of the\nmodeling of visual scenes and the inference of scene representations; then\nprovide benchmarks, including an open source toolbox to reproduce the benchmark\nexperiments, of representative methods that consider the most extensively\nstudied problem setting and form the foundation for other methods; and finally\ndiscuss the limitations of existing methods and future directions of this\nresearch topic.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2202.07135.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Compositional Scene Representation Learning via Reconstruction: A Survey"}],"highlightedSummary":[{"type":"highlight","text":"Visual scenes are composed of visual concepts and have the property of\ncombinatorial explosion."},{"type":"text","text":" An important reason for humans to efficiently learn\nfrom diverse visual scenes is the ability of compositional perception, and it\nis desirable for artificial intelligence to have similar abilities.\nCompositional scene representation learning is a task that enables such\nabilities. In recent years, various methods have been proposed to apply deep\nneural networks, which have been proven to be advantageous in representation\nlearning, to learn compositional scene representations via reconstruction,\nadvancing this research direction into the deep learning era. Learning via\nreconstruction is advantageous because it may utilize massive unlabeled data\nand avoid costly and laborious data annotation. In this survey, we first\noutline the current progress on reconstruction-based compositional scene\nrepresentation learning with deep neural networks, including development\nhistory and categorizations of existing methods from the perspectives of the\nmodeling of visual scenes and the inference of scene representations; then\nprovide benchmarks, including an open source toolbox to reproduce the benchmark\nexperiments, of representative methods that consider the most extensively\nstudied problem setting and form the foundation for other methods; and finally\ndiscuss the limitations of existing methods and future directions of this\nresearch topic."}]},{"paper":{"id":"2404.02790","authors":[{"_id":"661ced01439dad90fc2d6147","user":{"_id":"63d44162ff1384ce6c636dce","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1674854731168-noauth.jpeg","isPro":false,"fullname":"Petru-Daniel Tudosiu","user":"danieltudosiu","type":"user","name":"danieltudosiu"},"name":"Petru-Daniel Tudosiu","status":"claimed_verified","statusLastChangedAt":"2024-06-19T20:21:47.000Z","hidden":false},{"_id":"661ced01439dad90fc2d6148","name":"Yongxin Yang","hidden":false},{"_id":"661ced01439dad90fc2d6149","name":"Shifeng Zhang","hidden":false},{"_id":"661ced01439dad90fc2d614a","name":"Fei Chen","hidden":false},{"_id":"661ced01439dad90fc2d614b","name":"Steven McDonagh","hidden":false},{"_id":"661ced01439dad90fc2d614c","user":{"_id":"6480af33c0d3c70316d2a333","avatarUrl":"/avatars/95ec51e5408888d33b07696301ad34b6.svg","isPro":false,"fullname":"Gerasimos (Makis) Lampouras","user":"glampouras","type":"user","name":"glampouras"},"name":"Gerasimos Lampouras","status":"claimed_verified","statusLastChangedAt":"2024-07-31T12:49:38.860Z","hidden":false},{"_id":"661ced01439dad90fc2d614d","name":"Ignacio Iacobacci","hidden":false},{"_id":"661ced01439dad90fc2d614e","name":"Sarah Parisot","hidden":false}],"publishedAt":"2024-04-03T14:58:00.000Z","title":"MULAN: A Multi Layer Annotated Dataset for Controllable Text-to-Image\n  Generation","summary":"Text-to-image generation has achieved astonishing results, yet precise\nspatial controllability and prompt fidelity remain highly challenging. This\nlimitation is typically addressed through cumbersome prompt engineering, scene\nlayout conditioning, or image editing techniques which often require hand drawn\nmasks. Nonetheless, pre-existing works struggle to take advantage of the\nnatural instance-level compositionality of scenes due to the typically flat\nnature of rasterized RGB output images. Towards adressing this challenge, we\nintroduce MuLAn: a novel dataset comprising over 44K MUlti-Layer ANnotations of\nRGB images as multilayer, instance-wise RGBA decompositions, and over 100K\ninstance images. To build MuLAn, we developed a training free pipeline which\ndecomposes a monocular RGB image into a stack of RGBA layers comprising of\nbackground and isolated instances. We achieve this through the use of\npretrained general-purpose models, and by developing three modules: image\ndecomposition for instance discovery and extraction, instance completion to\nreconstruct occluded areas, and image re-assembly. We use our pipeline to\ncreate MuLAn-COCO and MuLAn-LAION datasets, which contain a variety of image\ndecompositions in terms of style, composition and complexity. With MuLAn, we\nprovide the first photorealistic resource providing instance decomposition and\nocclusion information for high quality images, opening up new avenues for\ntext-to-image generative AI research. With this, we aim to encourage the\ndevelopment of novel generation and editing technology, in particular\nlayer-wise solutions. MuLAn data resources are available at\nhttps://MuLAn-dataset.github.io/.","upvotes":0,"discussionId":"661ced04439dad90fc2d61ec","ai_summary":"MuLAn is a novel dataset with over 44K Multi-Layer Annotations of RGB images, enabling precise spatial controllability and prompt fidelity in text-to-image generation through instance-level compositionality.","ai_keywords":["MuLAn","RGB images","multi-layer","RGBA","instance-wise","pretrained general-purpose models","image decomposition","instance discovery","instance extraction","instance completion","image re-assembly","MuLAn-COCO","MuLAn-LAION"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-04-03T10:58:00.000Z","title":"MULAN: A Multi Layer Annotated Dataset for Controllable Text-to-Image\n  Generation","summary":"Text-to-image generation has achieved astonishing results, yet precise\nspatial controllability and prompt fidelity remain highly challenging. This\nlimitation is typically addressed through cumbersome prompt engineering, scene\nlayout conditioning, or image editing techniques which often require hand drawn\nmasks. Nonetheless, pre-existing works struggle to take advantage of the\nnatural instance-level compositionality of scenes due to the typically flat\nnature of rasterized RGB output images. Towards adressing this challenge, we\nintroduce MuLAn: a novel dataset comprising over 44K MUlti-Layer ANnotations of\nRGB images as multilayer, instance-wise RGBA decompositions, and over 100K\ninstance images. To build MuLAn, we developed a training free pipeline which\ndecomposes a monocular RGB image into a stack of RGBA layers comprising of\nbackground and isolated instances. We achieve this through the use of\npretrained general-purpose models, and by developing three modules: image\ndecomposition for instance discovery and extraction, instance completion to\nreconstruct occluded areas, and image re-assembly. We use our pipeline to\ncreate MuLAn-COCO and MuLAn-LAION datasets, which contain a variety of image\ndecompositions in terms of style, composition and complexity. With MuLAn, we\nprovide the first photorealistic resource providing instance decomposition and\nocclusion information for high quality images, opening up new avenues for\ntext-to-image generative AI research. With this, we aim to encourage the\ndevelopment of novel generation and editing technology, in particular\nlayer-wise solutions. MuLAn data resources are available at\nhttps://MuLAn-dataset.github.io/.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2404.02790.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"MULAN: A Multi Layer Annotated Dataset for Controllable Text-to-Image\n  Generation"}],"highlightedSummary":[{"type":"text","text":"Text-to-image generation has achieved astonishing results, yet precise\nspatial controllability and prompt fidelity remain highly challenging. This\nlimitation is typically addressed through cumbersome prompt engineering, scene\nlayout conditioning, or image editing techniques which often require hand drawn\nmasks. Nonetheless, pre-existing works struggle to take advantage of the\nnatural instance-level compositionality of scenes due to the typically flat\nnature of rasterized RGB output images. Towards adressing this challenge, we\nintroduce MuLAn: a novel dataset comprising over 44K MUlti-Layer ANnotations of\nRGB images as multilayer, instance-wise RGBA decompositions, and over 100K\ninstance images. To build MuLAn, we developed a training free pipeline which\ndecomposes a monocular RGB image into a stack of RGBA layers comprising of\nbackground and isolated instances. "},{"type":"highlight","text":"We achieve this through the use of\npretrained general-purpose models, and by developing three modules: image\ndecomposition for instance discovery and extraction, instance completion to\nreconstruct occluded areas, and image re-assembly."},{"type":"text","text":" We use our pipeline to\ncreate MuLAn-COCO and MuLAn-LAION datasets, which contain a variety of image\ndecompositions in terms of style, composition and complexity. With MuLAn, we\nprovide the first photorealistic resource providing instance decomposition and\nocclusion information for high quality images, opening up new avenues for\ntext-to-image generative AI research. With this, we aim to encourage the\ndevelopment of novel generation and editing technology, in particular\nlayer-wise solutions. MuLAn data resources are available at\nhttps://MuLAn-dataset.github.io/."}]},{"paper":{"id":"2310.10640","authors":[{"_id":"6539102b821b9e4a34929c7a","user":{"_id":"64eddf17e630ae6d575f6231","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64eddf17e630ae6d575f6231/W8oofVqStLEAK1Xi4fQTh.png","isPro":false,"fullname":"Hanan Gani","user":"hanangani","type":"user","name":"hanangani"},"name":"Hanan Gani","status":"claimed_verified","statusLastChangedAt":"2024-11-15T09:57:41.004Z","hidden":false},{"_id":"6539102b821b9e4a34929c7b","user":{"_id":"63fcae4f987f631186e50fba","avatarUrl":"/avatars/8ac9dff40d1b483b4f2f4e4cbd3f9088.svg","isPro":false,"fullname":"Shariq Farooq Bhat","user":"shariqfarooq","type":"user","name":"shariqfarooq"},"name":"Shariq Farooq Bhat","status":"claimed_verified","statusLastChangedAt":"2026-06-19T14:23:35.870Z","hidden":false},{"_id":"6539102b821b9e4a34929c7c","name":"Muzammal Naseer","hidden":false},{"_id":"6539102b821b9e4a34929c7d","name":"Salman Khan","hidden":false},{"_id":"6539102b821b9e4a34929c7e","name":"Peter Wonka","hidden":false}],"publishedAt":"2023-10-16T17:57:37.000Z","title":"LLM Blueprint: Enabling Text-to-Image Generation with Complex and\n  Detailed Prompts","summary":"Diffusion-based generative models have significantly advanced text-to-image\ngeneration but encounter challenges when processing lengthy and intricate text\nprompts describing complex scenes with multiple objects. While excelling in\ngenerating images from short, single-object descriptions, these models often\nstruggle to faithfully capture all the nuanced details within longer and more\nelaborate textual inputs. In response, we present a novel approach leveraging\nLarge Language Models (LLMs) to extract critical components from text prompts,\nincluding bounding box coordinates for foreground objects, detailed textual\ndescriptions for individual objects, and a succinct background context. These\ncomponents form the foundation of our layout-to-image generation model, which\noperates in two phases. The initial Global Scene Generation utilizes object\nlayouts and background context to create an initial scene but often falls short\nin faithfully representing object characteristics as specified in the prompts.\nTo address this limitation, we introduce an Iterative Refinement Scheme that\niteratively evaluates and refines box-level content to align them with their\ntextual descriptions, recomposing objects as needed to ensure consistency. Our\nevaluation on complex prompts featuring multiple objects demonstrates a\nsubstantial improvement in recall compared to baseline diffusion models. This\nis further validated by a user study, underscoring the efficacy of our approach\nin generating coherent and detailed scenes from intricate textual inputs.","upvotes":2,"discussionId":"65391030821b9e4a34929cfd","githubRepo":"https://github.com/hananshafi/llmblueprint","githubRepoAddedBy":"auto","ai_summary":"A novel approach using Large Language Models enhances diffusion-based generative models for text-to-image generation by extracting critical components and applying an iterative refinement scheme, improving recall and detail in complex scenes.","ai_keywords":["diffusion-based generative models","text-to-image generation","Large Language Models","bounding box coordinates","object layouts","global scene generation","iterative refinement scheme","recall"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":85},"publishedAt":"2023-10-16T13:57:37.000Z","title":"LLM Blueprint: Enabling Text-to-Image Generation with Complex and\n  Detailed Prompts","summary":"Diffusion-based generative models have significantly advanced text-to-image\ngeneration but encounter challenges when processing lengthy and intricate text\nprompts describing complex scenes with multiple objects. While excelling in\ngenerating images from short, single-object descriptions, these models often\nstruggle to faithfully capture all the nuanced details within longer and more\nelaborate textual inputs. In response, we present a novel approach leveraging\nLarge Language Models (LLMs) to extract critical components from text prompts,\nincluding bounding box coordinates for foreground objects, detailed textual\ndescriptions for individual objects, and a succinct background context. These\ncomponents form the foundation of our layout-to-image generation model, which\noperates in two phases. The initial Global Scene Generation utilizes object\nlayouts and background context to create an initial scene but often falls short\nin faithfully representing object characteristics as specified in the prompts.\nTo address this limitation, we introduce an Iterative Refinement Scheme that\niteratively evaluates and refines box-level content to align them with their\ntextual descriptions, recomposing objects as needed to ensure consistency. Our\nevaluation on complex prompts featuring multiple objects demonstrates a\nsubstantial improvement in recall compared to baseline diffusion models. This\nis further validated by a user study, underscoring the efficacy of our approach\nin generating coherent and detailed scenes from intricate textual inputs.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2310.10640.png","numComments":1,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"LLM Blueprint: Enabling Text-to-Image Generation with Complex and\n  Detailed Prompts"}],"highlightedSummary":[{"type":"text","text":"Diffusion-based generative models have significantly advanced text-to-image\ngeneration but encounter challenges when processing lengthy and intricate text\nprompts describing complex scenes with multiple objects. While excelling in\ngenerating images from short, single-object descriptions, these models often\nstruggle to faithfully capture all the nuanced details within longer and more\nelaborate textual inputs. In response, we present a novel approach leveraging\nLarge Language Models (LLMs) to extract critical components from text prompts,\nincluding bounding box coordinates for foreground objects, detailed textual\ndescriptions for individual objects, and a succinct background context. These\ncomponents form the foundation of our layout-to-image generation model, which\noperates in two phases. "},{"type":"highlight","text":"The initial Global Scene Generation utilizes object\nlayouts and background context to create an initial scene but often falls short\nin faithfully representing object characteristics as specified in the prompts."},{"type":"text","text":"\nTo address this limitation, we introduce an Iterative Refinement Scheme that\niteratively evaluates and refines box-level content to align them with their\ntextual descriptions, recomposing objects as needed to ensure consistency. Our\nevaluation on complex prompts featuring multiple objects demonstrates a\nsubstantial improvement in recall compared to baseline diffusion models. This\nis further validated by a user study, underscoring the efficacy of our approach\nin generating coherent and detailed scenes from intricate textual inputs."}]},{"paper":{"id":"2507.04152","authors":[{"_id":"697a4929df3e800774f13b0b","name":"Spencer Ramsey","hidden":false},{"_id":"697a4929df3e800774f13b0c","name":"Jeffrey Lee","hidden":false},{"_id":"697a4929df3e800774f13b0d","name":"Amina Grant","hidden":false}],"publishedAt":"2025-07-05T20:21:03.000Z","title":"LVLM-Composer's Explicit Planning for Image Generation","summary":"The burgeoning field of generative artificial intelligence has fundamentally reshaped our approach to content creation, with Large Vision-Language Models (LVLMs) standing at its forefront. While current LVLMs have demonstrated impressive capabilities in text-to-image generation, they often falter when confronted with complex textual descriptions demanding precise compositional understanding and visual planning. This limitation particularly impacts the accurate rendering of multiple objects, their attributes, spatial relationships, and specific poses within intricate scenes, as evidenced by benchmarks like LongBench-T2I. To address these challenges, we introduce LVLM-Composer, a novel 10-billion parameter scale LVLM specifically engineered for enhanced compositional image synthesis. Our method incorporates a Hierarchical Semantic Planning Module for structured prompt decomposition and a Fine-Grained Feature Alignment Mechanism for precise visual guidance during generation. We propose a multi-stage training paradigm, featuring Hierarchical Semantic-Visual Grounding Pre-training and Compositional Planning Reinforcement Learning with Self-Correction, to instill robust compositional reasoning. Extensive experiments on the LongBench-T2I benchmark, utilizing automatic evaluation by Gemini-2.0-Flash and InternVL3-78B, demonstrate LVLM-Composer's superior performance across critical compositional dimensions including object accuracy, composition fidelity, and pose accuracy, significantly outperforming state-of-the-art baselines. An in-depth ablation study further validates the indispensable contribution of our proposed modules, while human evaluations confirm the perceptual superiority of our generated images. LVLM-Composer represents a significant step towards truly controllable and compositionally accurate open-ended text-to-image generation.","upvotes":0,"discussionId":"697a4929df3e800774f13b0e","ai_summary":"LVLM-Composer enhances compositional image synthesis through hierarchical semantic planning and fine-grained feature alignment, achieving superior performance on complex text-to-image tasks.","ai_keywords":["Large Vision-Language Models","Hierarchical Semantic Planning Module","Fine-Grained Feature Alignment Mechanism","Hierarchical Semantic-Visual Grounding Pre-training","Compositional Planning Reinforcement Learning","Self-Correction"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-07-05T16:21:03.000Z","title":"LVLM-Composer's Explicit Planning for Image Generation","summary":"The burgeoning field of generative artificial intelligence has fundamentally reshaped our approach to content creation, with Large Vision-Language Models (LVLMs) standing at its forefront. While current LVLMs have demonstrated impressive capabilities in text-to-image generation, they often falter when confronted with complex textual descriptions demanding precise compositional understanding and visual planning. This limitation particularly impacts the accurate rendering of multiple objects, their attributes, spatial relationships, and specific poses within intricate scenes, as evidenced by benchmarks like LongBench-T2I. To address these challenges, we introduce LVLM-Composer, a novel 10-billion parameter scale LVLM specifically engineered for enhanced compositional image synthesis. Our method incorporates a Hierarchical Semantic Planning Module for structured prompt decomposition and a Fine-Grained Feature Alignment Mechanism for precise visual guidance during generation. We propose a multi-stage training paradigm, featuring Hierarchical Semantic-Visual Grounding Pre-training and Compositional Planning Reinforcement Learning with Self-Correction, to instill robust compositional reasoning. Extensive experiments on the LongBench-T2I benchmark, utilizing automatic evaluation by Gemini-2.0-Flash and InternVL3-78B, demonstrate LVLM-Composer's superior performance across critical compositional dimensions including object accuracy, composition fidelity, and pose accuracy, significantly outperforming state-of-the-art baselines. An in-depth ablation study further validates the indispensable contribution of our proposed modules, while human evaluations confirm the perceptual superiority of our generated images. LVLM-Composer represents a significant step towards truly controllable and compositionally accurate open-ended text-to-image generation.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2507.04152.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"LVLM-Composer's Explicit Planning for Image Generation"}],"highlightedSummary":[{"type":"text","text":"The burgeoning field of generative artificial intelligence has fundamentally reshaped our approach to content creation, with Large Vision-Language Models (LVLMs) standing at its forefront. While current LVLMs have demonstrated impressive capabilities in text-to-image generation, they often falter when confronted with complex textual descriptions demanding precise compositional understanding and visual planning. "},{"type":"highlight","text":"This limitation particularly impacts the accurate rendering of multiple objects, their attributes, spatial relationships, and specific poses within intricate scenes, as evidenced by benchmarks like LongBench-T2I."},{"type":"text","text":" To address these challenges, we introduce LVLM-Composer, a novel 10-billion parameter scale LVLM specifically engineered for enhanced compositional image synthesis. Our method incorporates a Hierarchical Semantic Planning Module for structured prompt decomposition and a Fine-Grained Feature Alignment Mechanism for precise visual guidance during generation. We propose a multi-stage training paradigm, featuring Hierarchical Semantic-Visual Grounding Pre-training and Compositional Planning Reinforcement Learning with Self-Correction, to instill robust compositional reasoning. Extensive experiments on the LongBench-T2I benchmark, utilizing automatic evaluation by Gemini-2.0-Flash and InternVL3-78B, demonstrate LVLM-Composer's superior performance across critical compositional dimensions including object accuracy, composition fidelity, and pose accuracy, significantly outperforming state-of-the-art baselines. An in-depth ablation study further validates the indispensable contribution of our proposed modules, while human evaluations confirm the perceptual superiority of our generated images. LVLM-Composer represents a significant step towards truly controllable and compositionally accurate open-ended text-to-image generation."}]},{"paper":{"id":"2512.08905","authors":[{"_id":"693fa4d5f516c69324681276","name":"Kaizhi Zheng","hidden":false},{"_id":"693fa4d5f516c69324681277","name":"Yue Fan","hidden":false},{"_id":"693fa4d5f516c69324681278","name":"Jing Gu","hidden":false},{"_id":"693fa4d5f516c69324681279","name":"Zishuo Xu","hidden":false},{"_id":"693fa4d5f516c6932468127a","name":"Xuehai He","hidden":false},{"_id":"693fa4d5f516c6932468127b","user":{"_id":"64679a226192d39142245e5e","avatarUrl":"/avatars/05abee0b6317f100923936ca2099e9eb.svg","isPro":false,"fullname":"Xin Eric Wang","user":"xw-eric","type":"user","name":"xw-eric"},"name":"Xin Eric Wang","status":"claimed_verified","statusLastChangedAt":"2026-02-06T18:57:44.021Z","hidden":false}],"publishedAt":"2025-12-09T18:44:21.000Z","title":"Self-Evolving 3D Scene Generation from a Single Image","summary":"Generating high-quality, textured 3D scenes from a single image remains a fundamental challenge in vision and graphics. Recent image-to-3D generators recover reasonable geometry from single views, but their object-centric training limits generalization to complex, large-scale scenes with faithful structure and texture. We present EvoScene, a self-evolving, training-free framework that progressively reconstructs complete 3D scenes from single images. The key idea is combining the complementary strengths of existing models: geometric reasoning from 3D generation models and visual knowledge from video generation models. Through three iterative stages--Spatial Prior Initialization, Visual-guided 3D Scene Mesh Generation, and Spatial-guided Novel View Generation--EvoScene alternates between 2D and 3D domains, gradually improving both structure and appearance. Experiments on diverse scenes demonstrate that EvoScene achieves superior geometric stability, view-consistent textures, and unseen-region completion compared to strong baselines, producing ready-to-use 3D meshes for practical applications.","upvotes":0,"discussionId":"693fa4d6f516c6932468127c","ai_summary":"EvoScene is a training-free framework that progressively reconstructs complete 3D scenes from single images by iteratively combining geometric reasoning and visual knowledge across 2D and 3D domains.","ai_keywords":["image-to-3D generation","3D scene reconstruction","geometric reasoning","video generation models","3D mesh generation","novel view generation","spatial prior initialization","visual-guided 3D scene generation","spatial-guided novel view generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-12-09T13:44:21.000Z","title":"Self-Evolving 3D Scene Generation from a Single Image","summary":"Generating high-quality, textured 3D scenes from a single image remains a fundamental challenge in vision and graphics. Recent image-to-3D generators recover reasonable geometry from single views, but their object-centric training limits generalization to complex, large-scale scenes with faithful structure and texture. We present EvoScene, a self-evolving, training-free framework that progressively reconstructs complete 3D scenes from single images. The key idea is combining the complementary strengths of existing models: geometric reasoning from 3D generation models and visual knowledge from video generation models. Through three iterative stages--Spatial Prior Initialization, Visual-guided 3D Scene Mesh Generation, and Spatial-guided Novel View Generation--EvoScene alternates between 2D and 3D domains, gradually improving both structure and appearance. Experiments on diverse scenes demonstrate that EvoScene achieves superior geometric stability, view-consistent textures, and unseen-region completion compared to strong baselines, producing ready-to-use 3D meshes for practical applications.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2512.08905.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Self-Evolving 3D Scene Generation from a Single Image"}],"highlightedSummary":[{"type":"highlight","text":"Generating high-quality, textured 3D scenes from a single image remains a fundamental challenge in vision and graphics."},{"type":"text","text":" Recent image-to-3D generators recover reasonable geometry from single views, but their object-centric training limits generalization to complex, large-scale scenes with faithful structure and texture. We present EvoScene, a self-evolving, training-free framework that progressively reconstructs complete 3D scenes from single images. The key idea is combining the complementary strengths of existing models: geometric reasoning from 3D generation models and visual knowledge from video generation models. Through three iterative stages--Spatial Prior Initialization, Visual-guided 3D Scene Mesh Generation, and Spatial-guided Novel View Generation--EvoScene alternates between 2D and 3D domains, gradually improving both structure and appearance. Experiments on diverse scenes demonstrate that EvoScene achieves superior geometric stability, view-consistent textures, and unseen-region completion compared to strong baselines, producing ready-to-use 3D meshes for practical applications."}]},{"paper":{"id":"2302.01330","authors":[{"_id":"65657567995cc49553e67ad2","name":"Zhaoxi Chen","hidden":false},{"_id":"65657567995cc49553e67ad3","name":"Guangcong Wang","hidden":false},{"_id":"65657567995cc49553e67ad4","name":"Ziwei Liu","hidden":false}],"publishedAt":"2023-02-02T18:59:16.000Z","title":"SceneDreamer: Unbounded 3D Scene Generation from 2D Image Collections","summary":"In this work, we present SceneDreamer, an unconditional generative model for\nunbounded 3D scenes, which synthesizes large-scale 3D landscapes from random\nnoise. Our framework is learned from in-the-wild 2D image collections only,\nwithout any 3D annotations. At the core of SceneDreamer is a principled\nlearning paradigm comprising 1) an efficient yet expressive 3D scene\nrepresentation, 2) a generative scene parameterization, and 3) an effective\nrenderer that can leverage the knowledge from 2D images. Our approach begins\nwith an efficient bird's-eye-view (BEV) representation generated from simplex\nnoise, which includes a height field for surface elevation and a semantic field\nfor detailed scene semantics. This BEV scene representation enables 1)\nrepresenting a 3D scene with quadratic complexity, 2) disentangled geometry and\nsemantics, and 3) efficient training. Moreover, we propose a novel generative\nneural hash grid to parameterize the latent space based on 3D positions and\nscene semantics, aiming to encode generalizable features across various scenes.\nLastly, a neural volumetric renderer, learned from 2D image collections through\nadversarial training, is employed to produce photorealistic images. Extensive\nexperiments demonstrate the effectiveness of SceneDreamer and superiority over\nstate-of-the-art methods in generating vivid yet diverse unbounded 3D worlds.","upvotes":0,"discussionId":"6565756b995cc49553e67bfd","githubRepo":"https://github.com/frozenburning/scenedreamer","githubRepoAddedBy":"auto","ai_summary":"SceneDreamer generates unbounded 3D scenes from 2D images using a novel 3D scene representation, generative neural hash grid, and adversarial neural volumetric renderer.","ai_keywords":["SceneDreamer","3D scene representation","bird's-eye-view","simplex noise","height field","semantic field","quadratic complexity","disentangled geometry","semantic","generative neural hash grid","latent space","neural volumetric renderer","adversarial training"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":661},"publishedAt":"2023-02-02T13:59:16.000Z","title":"SceneDreamer: Unbounded 3D Scene Generation from 2D Image Collections","summary":"In this work, we present SceneDreamer, an unconditional generative model for\nunbounded 3D scenes, which synthesizes large-scale 3D landscapes from random\nnoise. Our framework is learned from in-the-wild 2D image collections only,\nwithout any 3D annotations. At the core of SceneDreamer is a principled\nlearning paradigm comprising 1) an efficient yet expressive 3D scene\nrepresentation, 2) a generative scene parameterization, and 3) an effective\nrenderer that can leverage the knowledge from 2D images. Our approach begins\nwith an efficient bird's-eye-view (BEV) representation generated from simplex\nnoise, which includes a height field for surface elevation and a semantic field\nfor detailed scene semantics. This BEV scene representation enables 1)\nrepresenting a 3D scene with quadratic complexity, 2) disentangled geometry and\nsemantics, and 3) efficient training. Moreover, we propose a novel generative\nneural hash grid to parameterize the latent space based on 3D positions and\nscene semantics, aiming to encode generalizable features across various scenes.\nLastly, a neural volumetric renderer, learned from 2D image collections through\nadversarial training, is employed to produce photorealistic images. Extensive\nexperiments demonstrate the effectiveness of SceneDreamer and superiority over\nstate-of-the-art methods in generating vivid yet diverse unbounded 3D worlds.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2302.01330.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"SceneDreamer: Unbounded 3D Scene Generation from 2D Image Collections"}],"highlightedSummary":[{"type":"text","text":"In this work, we present SceneDreamer, an unconditional generative model for\nunbounded 3D scenes, which synthesizes large-scale 3D landscapes from random\nnoise. Our framework is learned from in-the-wild 2D image collections only,\nwithout any 3D annotations. At the core of SceneDreamer is a principled\nlearning paradigm comprising 1) an efficient yet expressive 3D scene\nrepresentation, 2) a generative scene parameterization, and 3) an effective\nrenderer that can leverage the knowledge from 2D images. Our approach begins\nwith an efficient bird's-eye-view (BEV) representation generated from simplex\nnoise, which includes a height field for surface elevation and a semantic field\nfor detailed scene semantics. This BEV scene representation enables 1)\nrepresenting a 3D scene with quadratic complexity, 2) disentangled geometry and\nsemantics, and 3) efficient training. Moreover, we propose a novel generative\nneural hash grid to parameterize the latent space based on 3D positions and\nscene semantics, aiming to encode generalizable features across various scenes.\nLastly, a neural volumetric renderer, learned from 2D image collections through\nadversarial training, is employed to produce photorealistic images. "},{"type":"highlight","text":"Extensive\nexperiments demonstrate the effectiveness of SceneDreamer and superiority over\nstate-of-the-art methods in generating vivid yet diverse unbounded 3D worlds."}]},{"paper":{"id":"2406.00687","authors":[{"_id":"667131d107bd06f8380c2f47","user":{"_id":"63c59c3a6d132b995fedface","avatarUrl":"/avatars/4e18b19e477cb683ce1ba3ae6ab77d8e.svg","isPro":false,"fullname":"Ohad rahamim","user":"ohad204","type":"user","name":"ohad204"},"name":"Ohad Rahamim","status":"claimed_verified","statusLastChangedAt":"2024-12-30T19:40:41.289Z","hidden":false},{"_id":"667131d107bd06f8380c2f48","name":"Hilit Segev","hidden":false},{"_id":"667131d107bd06f8380c2f49","name":"Idan Achituve","hidden":false},{"_id":"667131d107bd06f8380c2f4a","name":"Yuval Atzmon","hidden":false},{"_id":"667131d107bd06f8380c2f4b","name":"Yoni Kasten","hidden":false},{"_id":"667131d107bd06f8380c2f4c","name":"Gal Chechik","hidden":false}],"publishedAt":"2024-06-02T09:48:19.000Z","title":"Lay-A-Scene: Personalized 3D Object Arrangement Using Text-to-Image\n  Priors","summary":"Generating 3D visual scenes is at the forefront of visual generative AI, but\ncurrent 3D generation techniques struggle with generating scenes with multiple\nhigh-resolution objects. Here we introduce Lay-A-Scene, which solves the task\nof Open-set 3D Object Arrangement, effectively arranging unseen objects. Given\na set of 3D objects, the task is to find a plausible arrangement of these\nobjects in a scene. We address this task by leveraging pre-trained\ntext-to-image models. We personalize the model and explain how to generate\nimages of a scene that contains multiple predefined objects without neglecting\nany of them. Then, we describe how to infer the 3D poses and arrangement of\nobjects from a 2D generated image by finding a consistent projection of objects\nonto the 2D scene. We evaluate the quality of Lay-A-Scene using 3D objects from\nObjaverse and human raters and find that it often generates coherent and\nfeasible 3D object arrangements.","upvotes":0,"discussionId":"667131d807bd06f8380c3139","githubRepo":"https://github.com/ohad204/Lay-A-Scene","githubRepoAddedBy":"auto","ai_summary":"Lay-A-Scene uses text-to-image models to arrange 3D objects in a scene, generating coherent and feasible 3D object arrangements based on 2D projections.","ai_keywords":["text-to-image models","3D visual scenes","pre-trained models","3D objects","3D poses","2D generated image","2D projection","Objaverse","human raters","Open-set 3D Object Arrangement"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":9},"publishedAt":"2024-06-02T05:48:19.000Z","title":"Lay-A-Scene: Personalized 3D Object Arrangement Using Text-to-Image\n  Priors","summary":"Generating 3D visual scenes is at the forefront of visual generative AI, but\ncurrent 3D generation techniques struggle with generating scenes with multiple\nhigh-resolution objects. Here we introduce Lay-A-Scene, which solves the task\nof Open-set 3D Object Arrangement, effectively arranging unseen objects. Given\na set of 3D objects, the task is to find a plausible arrangement of these\nobjects in a scene. We address this task by leveraging pre-trained\ntext-to-image models. We personalize the model and explain how to generate\nimages of a scene that contains multiple predefined objects without neglecting\nany of them. Then, we describe how to infer the 3D poses and arrangement of\nobjects from a 2D generated image by finding a consistent projection of objects\nonto the 2D scene. We evaluate the quality of Lay-A-Scene using 3D objects from\nObjaverse and human raters and find that it often generates coherent and\nfeasible 3D object arrangements.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2406.00687.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Lay-A-Scene: Personalized 3D Object Arrangement Using Text-to-Image\n  Priors"}],"highlightedSummary":[{"type":"text","text":"Generating 3D visual scenes is at the forefront of visual generative AI, but\ncurrent 3D generation techniques struggle with generating scenes with multiple\nhigh-resolution objects. Here we introduce Lay-A-Scene, which solves the task\nof Open-set 3D Object Arrangement, effectively arranging unseen objects. Given\na set of 3D objects, the task is to find a plausible arrangement of these\nobjects in a scene. We address this task by leveraging pre-trained\ntext-to-image models. "},{"type":"highlight","text":"We personalize the model and explain how to generate\nimages of a scene that contains multiple predefined objects without neglecting\nany of them."},{"type":"text","text":" Then, we describe how to infer the 3D poses and arrangement of\nobjects from a 2D generated image by finding a consistent projection of objects\nonto the 2D scene. We evaluate the quality of Lay-A-Scene using 3D objects from\nObjaverse and human raters and find that it often generates coherent and\nfeasible 3D object arrangements."}]},{"paper":{"id":"2607.01708","authors":[{"_id":"6a7becba1653ef87c6af1c96","name":"Hyunjoon Park","hidden":false},{"_id":"6a7becba1653ef87c6af1c97","name":"Donghyeon Cho","hidden":false}],"publishedAt":"2026-07-02T00:00:00.000Z","title":"Consistent Scene Understanding in 3D Gaussian Splatting via Multi-Cue Mask Refinement","summary":"Reliable instance-level scene understanding is a fundamental prerequisite for object-level interactions and high-fidelity 3D representations. While current methods often leverage 2D foundation segmentation models to obtain these priors, their 2D-centric design typically yields fragmented masks and inconsistent predictions across different views. To address these issues, we propose a novel framework that produces consistent 2D instance masks to guide the optimization of 3D Gaussian Splatting (3DGS) feature fields. Our framework consists of three main stages. (1) Multi-Cue Extraction that generates synergistic semantic, geometric, and structural priors from input images. (2) Multi-Cue-Guided Mask Merging process that consolidates fragmented masks using a composite merge score derived from semantic, depth, and edge cues. (3) Cross-View Mask Matching that establishes globally consistent identity assignments across all viewpoints. By transforming viewpoint-specific segments into coherent 3D primitives, our approach enables stable 3D instance segmentation and effective downstream editing tasks. Experiments demonstrate that our method significantly improves cross-view consistency and segmentation stability over existing baselines while maintaining high-fidelity photometric reconstruction.","upvotes":0,"discussionId":"6a7becbb1653ef87c6af1c98","ai_summary":"The framework improves cross-view consistency of 2D instance masks to guide 3D Gaussian Splatting feature fields for stable 3D instance segmentation and editing.","ai_keywords":["3D Gaussian Splatting","instance segmentation","cross-view consistency","multi-cue extraction","mask merging","feature fields"],"ai_summary_model":"thinkingmachines/Inkling-Small"},"publishedAt":"2026-07-01T20:00:00.000Z","title":"Consistent Scene Understanding in 3D Gaussian Splatting via Multi-Cue Mask Refinement","summary":"Reliable instance-level scene understanding is a fundamental prerequisite for object-level interactions and high-fidelity 3D representations. While current methods often leverage 2D foundation segmentation models to obtain these priors, their 2D-centric design typically yields fragmented masks and inconsistent predictions across different views. To address these issues, we propose a novel framework that produces consistent 2D instance masks to guide the optimization of 3D Gaussian Splatting (3DGS) feature fields. Our framework consists of three main stages. (1) Multi-Cue Extraction that generates synergistic semantic, geometric, and structural priors from input images. (2) Multi-Cue-Guided Mask Merging process that consolidates fragmented masks using a composite merge score derived from semantic, depth, and edge cues. (3) Cross-View Mask Matching that establishes globally consistent identity assignments across all viewpoints. By transforming viewpoint-specific segments into coherent 3D primitives, our approach enables stable 3D instance segmentation and effective downstream editing tasks. Experiments demonstrate that our method significantly improves cross-view consistency and segmentation stability over existing baselines while maintaining high-fidelity photometric reconstruction.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2607.01708.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Consistent Scene Understanding in 3D Gaussian Splatting via Multi-Cue Mask Refinement"}],"highlightedSummary":[{"type":"highlight","text":"Reliable instance-level scene understanding is a fundamental prerequisite for object-level interactions and high-fidelity 3D representations."},{"type":"text","text":" While current methods often leverage 2D foundation segmentation models to obtain these priors, their 2D-centric design typically yields fragmented masks and inconsistent predictions across different views. To address these issues, we propose a novel framework that produces consistent 2D instance masks to guide the optimization of 3D Gaussian Splatting (3DGS) feature fields. Our framework consists of three main stages. (1) Multi-Cue Extraction that generates synergistic semantic, geometric, and structural priors from input images. (2) Multi-Cue-Guided Mask Merging process that consolidates fragmented masks using a composite merge score derived from semantic, depth, and edge cues. (3) Cross-View Mask Matching that establishes globally consistent identity assignments across all viewpoints. By transforming viewpoint-specific segments into coherent 3D primitives, our approach enables stable 3D instance segmentation and effective downstream editing tasks. Experiments demonstrate that our method significantly improves cross-view consistency and segmentation stability over existing baselines while maintaining high-fidelity photometric reconstruction."}]},{"paper":{"id":"2512.05272","authors":[{"_id":"6941484465f1e24a1178087f","user":{"_id":"662fcfeddeb646171f1df312","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/noauth/V1WdIbDyd20COpAAF70qf.jpeg","isPro":false,"fullname":"Berke Gökmen","user":"berkegokmen1","type":"user","name":"berkegokmen1"},"name":"Ahmet Berke Gokmen","status":"claimed_verified","statusLastChangedAt":"2025-12-16T14:10:23.454Z","hidden":false},{"_id":"6941484465f1e24a11780880","name":"Ajad Chhatkuli","hidden":false},{"_id":"6941484465f1e24a11780881","name":"Luc Van Gool","hidden":false},{"_id":"6941484465f1e24a11780882","name":"Danda Pani Paudel","hidden":false}],"publishedAt":"2025-12-04T21:51:47.000Z","title":"Inferring Compositional 4D Scenes without Ever Seeing One","summary":"Scenes in the real world are often composed of several static and dynamic objects. Capturing their 4-dimensional structures, composition and spatio-temporal configuration in-the-wild, though extremely interesting, is equally hard. Therefore, existing works often focus on one object at a time, while relying on some category-specific parametric shape model for dynamic objects. This can lead to inconsistent scene configurations, in addition to being limited to the modeled object categories. We propose COM4D (Compositional 4D), a method that consistently and jointly predicts the structure and spatio-temporal configuration of 4D/3D objects using only static multi-object or dynamic single object supervision. We achieve this by a carefully designed training of spatial and temporal attentions on 2D video input. The training is disentangled into learning from object compositions on the one hand, and single object dynamics throughout the video on the other, thus completely avoiding reliance on 4D compositional training data. At inference time, our proposed attention mixing mechanism combines these independently learned attentions, without requiring any 4D composition examples. By alternating between spatial and temporal reasoning, COM4D reconstructs complete and persistent 4D scenes with multiple interacting objects directly from monocular videos. Furthermore, COM4D provides state-of-the-art results in existing separate problems of 4D object and composed 3D reconstruction despite being purely data-driven.","upvotes":4,"discussionId":"6941484465f1e24a11780883","projectPage":"https://com4d.insait.ai","githubRepo":"https://github.com/insait-institute/COM4D","githubRepoAddedBy":"user","ai_summary":"COM4D predicts the structure and spatio-temporal configuration of 4D/3D objects from 2D video without using 4D compositional training data, achieving state-of-the-art results in 4D object and composed 3D reconstruction.","ai_keywords":["4D structures","spatio-temporal configuration","category-specific parametric shape model","COM4D","Compositional 4D","spatial attention","temporal attention","attention mixing mechanism","spatial reasoning","temporal reasoning","4D reconstruction","3D reconstruction"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":28,"organization":{"_id":"6585b575110c3eb77dabaa93","name":"INSAIT-Institute","fullname":"Institute for Computer Science, Artificial intelligence and Technology ","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/64f1a0700af832a73d0f3e6f/KwpuATq29U2-Fu55OvUHR.png"}},"publishedAt":"2025-12-04T16:51:47.000Z","title":"Inferring Compositional 4D Scenes without Ever Seeing One","summary":"Scenes in the real world are often composed of several static and dynamic objects. Capturing their 4-dimensional structures, composition and spatio-temporal configuration in-the-wild, though extremely interesting, is equally hard. Therefore, existing works often focus on one object at a time, while relying on some category-specific parametric shape model for dynamic objects. This can lead to inconsistent scene configurations, in addition to being limited to the modeled object categories. We propose COM4D (Compositional 4D), a method that consistently and jointly predicts the structure and spatio-temporal configuration of 4D/3D objects using only static multi-object or dynamic single object supervision. We achieve this by a carefully designed training of spatial and temporal attentions on 2D video input. The training is disentangled into learning from object compositions on the one hand, and single object dynamics throughout the video on the other, thus completely avoiding reliance on 4D compositional training data. At inference time, our proposed attention mixing mechanism combines these independently learned attentions, without requiring any 4D composition examples. By alternating between spatial and temporal reasoning, COM4D reconstructs complete and persistent 4D scenes with multiple interacting objects directly from monocular videos. Furthermore, COM4D provides state-of-the-art results in existing separate problems of 4D object and composed 3D reconstruction despite being purely data-driven.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2512.05272.png","numComments":2,"upvoted":false,"organization":{"_id":"6585b575110c3eb77dabaa93","name":"INSAIT-Institute","fullname":"Institute for Computer Science, Artificial intelligence and Technology ","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/64f1a0700af832a73d0f3e6f/KwpuATq29U2-Fu55OvUHR.png"},"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Inferring Compositional 4D Scenes without Ever Seeing One"}],"highlightedSummary":[{"type":"highlight","text":"Scenes in the real world are often composed of several static and dynamic objects."},{"type":"text","text":" Capturing their 4-dimensional structures, composition and spatio-temporal configuration in-the-wild, though extremely interesting, is equally hard. Therefore, existing works often focus on one object at a time, while relying on some category-specific parametric shape model for dynamic objects. This can lead to inconsistent scene configurations, in addition to being limited to the modeled object categories. We propose COM4D (Compositional 4D), a method that consistently and jointly predicts the structure and spatio-temporal configuration of 4D/3D objects using only static multi-object or dynamic single object supervision. We achieve this by a carefully designed training of spatial and temporal attentions on 2D video input. The training is disentangled into learning from object compositions on the one hand, and single object dynamics throughout the video on the other, thus completely avoiding reliance on 4D compositional training data. At inference time, our proposed attention mixing mechanism combines these independently learned attentions, without requiring any 4D composition examples. By alternating between spatial and temporal reasoning, COM4D reconstructs complete and persistent 4D scenes with multiple interacting objects directly from monocular videos. Furthermore, COM4D provides state-of-the-art results in existing separate problems of 4D object and composed 3D reconstruction despite being purely data-driven."}]},{"paper":{"id":"2510.21682","authors":[{"_id":"68fecebf6cdff8b857f470e5","name":"Sikuang Li","hidden":false},{"_id":"68fecebf6cdff8b857f470e6","name":"Chen Yang","hidden":false},{"_id":"68fecebf6cdff8b857f470e7","user":{"_id":"652cf49cc6857682d3168f5a","avatarUrl":"/avatars/9ccffa04832429aad37320a301ea8e36.svg","isPro":false,"fullname":"Jiemin Fang","user":"JieminFang","type":"user","name":"JieminFang"},"name":"Jiemin Fang","status":"claimed_verified","statusLastChangedAt":"2025-10-27T10:27:40.128Z","hidden":false},{"_id":"68fecebf6cdff8b857f470e8","user":{"_id":"64e5ee4b93d04e3439f4e988","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64e5ee4b93d04e3439f4e988/g-THJvunoSPUyQ2v_fChU.jpeg","isPro":false,"fullname":"taoranyi","user":"thewhole","type":"user","name":"thewhole"},"name":"Taoran Yi","status":"claimed_verified","statusLastChangedAt":"2025-10-29T12:50:27.836Z","hidden":false},{"_id":"68fecebf6cdff8b857f470e9","name":"Jia Lu","hidden":false},{"_id":"68fecebf6cdff8b857f470ea","name":"Jiazhong Cen","hidden":false},{"_id":"68fecebf6cdff8b857f470eb","name":"Lingxi Xie","hidden":false},{"_id":"68fecebf6cdff8b857f470ec","name":"Wei Shen","hidden":false},{"_id":"68fecebf6cdff8b857f470ed","name":"Qi Tian","hidden":false}],"publishedAt":"2025-10-24T17:39:52.000Z","title":"WorldGrow: Generating Infinite 3D World","summary":"We tackle the challenge of generating the infinitely extendable 3D world --\nlarge, continuous environments with coherent geometry and realistic appearance.\nExisting methods face key challenges: 2D-lifting approaches suffer from\ngeometric and appearance inconsistencies across views, 3D implicit\nrepresentations are hard to scale up, and current 3D foundation models are\nmostly object-centric, limiting their applicability to scene-level generation.\nOur key insight is leveraging strong generation priors from pre-trained 3D\nmodels for structured scene block generation. To this end, we propose\nWorldGrow, a hierarchical framework for unbounded 3D scene synthesis. Our\nmethod features three core components: (1) a data curation pipeline that\nextracts high-quality scene blocks for training, making the 3D structured\nlatent representations suitable for scene generation; (2) a 3D block inpainting\nmechanism that enables context-aware scene extension; and (3) a coarse-to-fine\ngeneration strategy that ensures both global layout plausibility and local\ngeometric/textural fidelity. Evaluated on the large-scale 3D-FRONT dataset,\nWorldGrow achieves SOTA performance in geometry reconstruction, while uniquely\nsupporting infinite scene generation with photorealistic and structurally\nconsistent outputs. These results highlight its capability for constructing\nlarge-scale virtual environments and potential for building future world\nmodels.","upvotes":42,"discussionId":"68fecebf6cdff8b857f470ee","projectPage":"https://world-grow.github.io/","githubRepo":"https://github.com/world-grow/WorldGrow","githubRepoAddedBy":"user","ai_summary":"WorldGrow, a hierarchical framework, generates large, continuous 3D environments with coherent geometry and realistic appearance using pre-trained 3D models and a coarse-to-fine generation strategy.","ai_keywords":["3D implicit representations","3D foundation models","scene block generation","data curation pipeline","3D block inpainting","coarse-to-fine generation","geometry reconstruction","3D-FRONT dataset","world models"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":501},"publishedAt":"2025-10-24T13:39:52.000Z","title":"WorldGrow: Generating Infinite 3D World","summary":"We tackle the challenge of generating the infinitely extendable 3D world --\nlarge, continuous environments with coherent geometry and realistic appearance.\nExisting methods face key challenges: 2D-lifting approaches suffer from\ngeometric and appearance inconsistencies across views, 3D implicit\nrepresentations are hard to scale up, and current 3D foundation models are\nmostly object-centric, limiting their applicability to scene-level generation.\nOur key insight is leveraging strong generation priors from pre-trained 3D\nmodels for structured scene block generation. To this end, we propose\nWorldGrow, a hierarchical framework for unbounded 3D scene synthesis. Our\nmethod features three core components: (1) a data curation pipeline that\nextracts high-quality scene blocks for training, making the 3D structured\nlatent representations suitable for scene generation; (2) a 3D block inpainting\nmechanism that enables context-aware scene extension; and (3) a coarse-to-fine\ngeneration strategy that ensures both global layout plausibility and local\ngeometric/textural fidelity. Evaluated on the large-scale 3D-FRONT dataset,\nWorldGrow achieves SOTA performance in geometry reconstruction, while uniquely\nsupporting infinite scene generation with photorealistic and structurally\nconsistent outputs. These results highlight its capability for constructing\nlarge-scale virtual environments and potential for building future world\nmodels.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2510.21682.png","numComments":3,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"WorldGrow: Generating Infinite 3D World"}],"highlightedSummary":[{"type":"text","text":"We tackle the challenge of generating the infinitely extendable 3D world --\nlarge, continuous environments with coherent geometry and realistic appearance.\nExisting methods face key challenges: 2D-lifting approaches suffer from\ngeometric and appearance inconsistencies across views, 3D implicit\nrepresentations are hard to scale up, and current 3D foundation models are\nmostly object-centric, limiting their applicability to scene-level generation.\nOur key insight is leveraging strong generation priors from pre-trained 3D\nmodels for structured scene block generation. To this end, we propose\nWorldGrow, a hierarchical framework for unbounded 3D scene synthesis. "},{"type":"highlight","text":"Our\nmethod features three core components: (1) a data curation pipeline that\nextracts high-quality scene blocks for training, making the 3D structured\nlatent representations suitable for scene generation; (2) a 3D block inpainting\nmechanism that enables context-aware scene extension; and (3) a coarse-to-fine\ngeneration strategy that ensures both global layout plausibility and local\ngeometric/textural fidelity."},{"type":"text","text":" Evaluated on the large-scale 3D-FRONT dataset,\nWorldGrow achieves SOTA performance in geometry reconstruction, while uniquely\nsupporting infinite scene generation with photorealistic and structurally\nconsistent outputs. These results highlight its capability for constructing\nlarge-scale virtual environments and potential for building future world\nmodels."}]},{"paper":{"id":"2402.16936","authors":[{"_id":"65deb5036d161782405dd172","user":{"_id":"64caa93391726743360ab1f0","avatarUrl":"/avatars/992cb5aaf42d21be03a998656bb485d7.svg","isPro":false,"fullname":"Dave Epstein","user":"daveepstein","type":"user","name":"daveepstein"},"name":"Dave Epstein","status":"admin_assigned","statusLastChangedAt":"2024-02-28T12:31:31.787Z","hidden":false},{"_id":"65deb5036d161782405dd173","user":{"_id":"63213768d2d45f315183abc3","avatarUrl":"/avatars/521f52f430b73e514afb06e3a9d6779d.svg","isPro":false,"fullname":"Ben Poole","user":"doinkda","type":"user","name":"doinkda"},"name":"Ben Poole","status":"admin_assigned","statusLastChangedAt":"2024-02-28T12:31:37.752Z","hidden":false},{"_id":"65deb5036d161782405dd174","name":"Ben Mildenhall","hidden":false},{"_id":"65deb5036d161782405dd175","name":"Alexei A. Efros","hidden":false},{"_id":"65deb5036d161782405dd176","user":{"_id":"62f6e2792e53c2efd33faa92","avatarUrl":"/avatars/85d94b4022577747b8d2d10a82c2f3c7.svg","isPro":false,"fullname":"Aleksander Holynski","user":"holynski","type":"user","name":"holynski"},"name":"Aleksander Holynski","status":"admin_assigned","statusLastChangedAt":"2024-02-28T12:31:57.221Z","hidden":false}],"publishedAt":"2024-02-26T18:54:15.000Z","title":"Disentangled 3D Scene Generation with Layout Learning","summary":"We introduce a method to generate 3D scenes that are disentangled into their\ncomponent objects. This disentanglement is unsupervised, relying only on the\nknowledge of a large pretrained text-to-image model. Our key insight is that\nobjects can be discovered by finding parts of a 3D scene that, when rearranged\nspatially, still produce valid configurations of the same scene. Concretely,\nour method jointly optimizes multiple NeRFs from scratch - each representing\nits own object - along with a set of layouts that composite these objects into\nscenes. We then encourage these composited scenes to be in-distribution\naccording to the image generator. We show that despite its simplicity, our\napproach successfully generates 3D scenes decomposed into individual objects,\nenabling new capabilities in text-to-3D content creation. For results and an\ninteractive demo, see our project page at https://dave.ml/layoutlearning/","upvotes":11,"discussionId":"65deb5066d161782405dd225","ai_summary":"Unsupervised disentanglement of 3D scenes into component objects using pretrained text-to-image models and joint optimization of NeRFs and layouts.","ai_keywords":["NeRFs","in-distribution","text-to-3D","scene decomposition","object discovery"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-02-26T13:54:15.000Z","title":"Disentangled 3D Scene Generation with Layout Learning","summary":"We introduce a method to generate 3D scenes that are disentangled into their\ncomponent objects. This disentanglement is unsupervised, relying only on the\nknowledge of a large pretrained text-to-image model. Our key insight is that\nobjects can be discovered by finding parts of a 3D scene that, when rearranged\nspatially, still produce valid configurations of the same scene. Concretely,\nour method jointly optimizes multiple NeRFs from scratch - each representing\nits own object - along with a set of layouts that composite these objects into\nscenes. We then encourage these composited scenes to be in-distribution\naccording to the image generator. We show that despite its simplicity, our\napproach successfully generates 3D scenes decomposed into individual objects,\nenabling new capabilities in text-to-3D content creation. For results and an\ninteractive demo, see our project page at https://dave.ml/layoutlearning/","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2402.16936.png","numComments":1,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Disentangled 3D Scene Generation with Layout Learning"}],"highlightedSummary":[{"type":"text","text":"We introduce a method to generate 3D scenes that are disentangled into their\ncomponent objects. This disentanglement is unsupervised, relying only on the\nknowledge of a large pretrained text-to-image model. Our key insight is that\nobjects can be discovered by finding parts of a 3D scene that, when rearranged\nspatially, still produce valid configurations of the same scene. Concretely,\nour method jointly optimizes multiple NeRFs from scratch - each representing\nits own object - along with a set of layouts that composite these objects into\nscenes. We then encourage these composited scenes to be in-distribution\naccording to the image generator. "},{"type":"highlight","text":"We show that despite its simplicity, our\napproach successfully generates 3D scenes decomposed into individual objects,\nenabling new capabilities in text-to-3D content creation."},{"type":"text","text":" For results and an\ninteractive demo, see our project page at https://dave.ml/layoutlearning/"}]},{"paper":{"id":"2003.14166","authors":[{"_id":"667cd08e0204c59594ae57cf","user":{"_id":"642f99079b2484d7d857341b","avatarUrl":"/avatars/01965cc5a5dbe9c08025a51973462a6a.svg","isPro":false,"fullname":"Sai Rajeswar","user":"rajeswarsai","type":"user","name":"rajeswarsai"},"name":"Sai Rajeswar","status":"claimed_verified","statusLastChangedAt":"2024-06-27T07:31:20.887Z","hidden":false},{"_id":"667cd08e0204c59594ae57d0","name":"Fahim Mannan","hidden":false},{"_id":"667cd08e0204c59594ae57d1","name":"Florian Golemo","hidden":false},{"_id":"667cd08e0204c59594ae57d2","name":"Jérôme Parent-Lévesque","hidden":false},{"_id":"667cd08e0204c59594ae57d3","user":{"_id":"62f6691f329d4d014d1b4087","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1660315866736-noauth.jpeg","isPro":false,"fullname":"David Vazquez","user":"dvazquez","type":"user","name":"dvazquez"},"name":"David Vazquez","status":"claimed_verified","statusLastChangedAt":"2025-05-08T09:00:53.365Z","hidden":false},{"_id":"667cd08e0204c59594ae57d4","name":"Derek Nowrouzezahrai","hidden":false},{"_id":"667cd08e0204c59594ae57d5","name":"Aaron Courville","hidden":false}],"publishedAt":"2020-03-23T03:01:34.000Z","title":"Pix2Shape: Towards Unsupervised Learning of 3D Scenes from Images using\n  a View-based Representation","summary":"We infer and generate three-dimensional (3D) scene information from a single\ninput image and without supervision. This problem is under-explored, with most\nprior work relying on supervision from, e.g., 3D ground-truth, multiple images\nof a scene, image silhouettes or key-points. We propose Pix2Shape, an approach\nto solve this problem with four components: (i) an encoder that infers the\nlatent 3D representation from an image, (ii) a decoder that generates an\nexplicit 2.5D surfel-based reconstruction of a scene from the latent code (iii)\na differentiable renderer that synthesizes a 2D image from the surfel\nrepresentation, and (iv) a critic network trained to discriminate between\nimages generated by the decoder-renderer and those from a training\ndistribution. Pix2Shape can generate complex 3D scenes that scale with the\nview-dependent on-screen resolution, unlike representations that capture\nworld-space resolution, i.e., voxels or meshes. We show that Pix2Shape learns a\nconsistent scene representation in its encoded latent space and that the\ndecoder can then be applied to this latent representation in order to\nsynthesize the scene from a novel viewpoint. We evaluate Pix2Shape with\nexperiments on the ShapeNet dataset as well as on a novel benchmark we\ndeveloped, called 3D-IQTT, to evaluate models based on their ability to enable\n3d spatial reasoning. Qualitative and quantitative evaluation demonstrate\nPix2Shape's ability to solve scene reconstruction, generation, and\nunderstanding tasks.","upvotes":0,"discussionId":"667cd0900204c59594ae5892","githubRepo":"https://github.com/rajeswar18/pix2shape","githubRepoAddedBy":"auto","ai_summary":"Pix2Shape infers and generates 3D scene information from a single image using a latent 3D representation and achieves view-dependent 2.5D surfel-based reconstructions.","ai_keywords":["encoder","decoder","latent 3D representation","2.5D surfel-based reconstruction","differentiable renderer","critic network","Pix2Shape","ShapeNet dataset","3D-IQTT","view-dependent resolution","3D spatial reasoning"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":29},"publishedAt":"2020-03-22T23:01:34.000Z","title":"Pix2Shape: Towards Unsupervised Learning of 3D Scenes from Images using\n  a View-based Representation","summary":"We infer and generate three-dimensional (3D) scene information from a single\ninput image and without supervision. This problem is under-explored, with most\nprior work relying on supervision from, e.g., 3D ground-truth, multiple images\nof a scene, image silhouettes or key-points. We propose Pix2Shape, an approach\nto solve this problem with four components: (i) an encoder that infers the\nlatent 3D representation from an image, (ii) a decoder that generates an\nexplicit 2.5D surfel-based reconstruction of a scene from the latent code (iii)\na differentiable renderer that synthesizes a 2D image from the surfel\nrepresentation, and (iv) a critic network trained to discriminate between\nimages generated by the decoder-renderer and those from a training\ndistribution. Pix2Shape can generate complex 3D scenes that scale with the\nview-dependent on-screen resolution, unlike representations that capture\nworld-space resolution, i.e., voxels or meshes. We show that Pix2Shape learns a\nconsistent scene representation in its encoded latent space and that the\ndecoder can then be applied to this latent representation in order to\nsynthesize the scene from a novel viewpoint. We evaluate Pix2Shape with\nexperiments on the ShapeNet dataset as well as on a novel benchmark we\ndeveloped, called 3D-IQTT, to evaluate models based on their ability to enable\n3d spatial reasoning. Qualitative and quantitative evaluation demonstrate\nPix2Shape's ability to solve scene reconstruction, generation, and\nunderstanding tasks.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2003.14166.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Pix2Shape: Towards Unsupervised Learning of 3D Scenes from Images using\n  a View-based Representation"}],"highlightedSummary":[{"type":"highlight","text":"We infer and generate three-dimensional (3D) scene information from a single\ninput image and without supervision."},{"type":"text","text":" This problem is under-explored, with most\nprior work relying on supervision from, e.g., 3D ground-truth, multiple images\nof a scene, image silhouettes or key-points. We propose Pix2Shape, an approach\nto solve this problem with four components: (i) an encoder that infers the\nlatent 3D representation from an image, (ii) a decoder that generates an\nexplicit 2.5D surfel-based reconstruction of a scene from the latent code (iii)\na differentiable renderer that synthesizes a 2D image from the surfel\nrepresentation, and (iv) a critic network trained to discriminate between\nimages generated by the decoder-renderer and those from a training\ndistribution. Pix2Shape can generate complex 3D scenes that scale with the\nview-dependent on-screen resolution, unlike representations that capture\nworld-space resolution, i.e., voxels or meshes. We show that Pix2Shape learns a\nconsistent scene representation in its encoded latent space and that the\ndecoder can then be applied to this latent representation in order to\nsynthesize the scene from a novel viewpoint. We evaluate Pix2Shape with\nexperiments on the ShapeNet dataset as well as on a novel benchmark we\ndeveloped, called 3D-IQTT, to evaluate models based on their ability to enable\n3d spatial reasoning. Qualitative and quantitative evaluation demonstrate\nPix2Shape's ability to solve scene reconstruction, generation, and\nunderstanding tasks."}]},{"paper":{"id":"2609.10531","authors":[{"_id":"6aabc31ad880715a3d023eb4","name":"Jerred Chen","hidden":false},{"_id":"6aabc31ad880715a3d023eb5","user":{"_id":"6a1ff199c30665929c7c5ccb","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/noauth/XykYIkOjNjFwZsAJWXeCX.png","isPro":false,"fullname":"Simon Weber","user":"SimonWeber","type":"user","name":"SimonWeber"},"name":"Simon Weber","status":"claimed_verified","statusLastChangedAt":"2026-09-17T16:45:04.565Z","hidden":false},{"_id":"6aabc31ad880715a3d023eb6","name":"Ronald Clark","hidden":false}],"publishedAt":"2026-09-09T00:00:00.000Z","title":"Guiding Image-to-3D Generation with Test-Time Partial Observations","summary":"Image-to-3D models can generate visually compelling 3D assets from a single RGB image, but their geometry is often only loosely constrained by the available observations, limiting their use in applications that require geometric fidelity. In many real-world settings, however, partial geometric observations of the object may be available at test time. We introduce a training-free framework for incorporating such evidence into pretrained image-to-3D generative models without retraining or finetuning. To do this, we guide generation using a ray-consistent observation likelihood defined over the model's occupancy representation, combining surface occupancy and free-space evidence. Applied to SAM 3D and its multi-view extension, our approach substantially improves geometric fidelity across different levels of observability, as well as visual quality. Our results demonstrate that pretrained image-to-3D models can effectively integrate partial geometric observations through explicit test-time guidance, complementing their learned generative priors without modifying the underlying model.","upvotes":0,"discussionId":"6aabc31ad880715a3d023eb7"},"publishedAt":"2026-09-08T20:00:00.000Z","title":"Guiding Image-to-3D Generation with Test-Time Partial Observations","summary":"Image-to-3D models can generate visually compelling 3D assets from a single RGB image, but their geometry is often only loosely constrained by the available observations, limiting their use in applications that require geometric fidelity. In many real-world settings, however, partial geometric observations of the object may be available at test time. We introduce a training-free framework for incorporating such evidence into pretrained image-to-3D generative models without retraining or finetuning. To do this, we guide generation using a ray-consistent observation likelihood defined over the model's occupancy representation, combining surface occupancy and free-space evidence. Applied to SAM 3D and its multi-view extension, our approach substantially improves geometric fidelity across different levels of observability, as well as visual quality. Our results demonstrate that pretrained image-to-3D models can effectively integrate partial geometric observations through explicit test-time guidance, complementing their learned generative priors without modifying the underlying model.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2609.10531.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Guiding Image-to-3D Generation with Test-Time Partial Observations"}],"highlightedSummary":[{"type":"highlight","text":"Image-to-3D models can generate visually compelling 3D assets from a single RGB image, but their geometry is often only loosely constrained by the available observations, limiting their use in applications that require geometric fidelity."},{"type":"text","text":" In many real-world settings, however, partial geometric observations of the object may be available at test time. We introduce a training-free framework for incorporating such evidence into pretrained image-to-3D generative models without retraining or finetuning. To do this, we guide generation using a ray-consistent observation likelihood defined over the model's occupancy representation, combining surface occupancy and free-space evidence. Applied to SAM 3D and its multi-view extension, our approach substantially improves geometric fidelity across different levels of observability, as well as visual quality. Our results demonstrate that pretrained image-to-3D models can effectively integrate partial geometric observations through explicit test-time guidance, complementing their learned generative priors without modifying the underlying model."}]},{"paper":{"id":"2312.10671","authors":[{"_id":"658187935cfd2c169a6659f9","user":{"_id":"64c108086b2f05ae643aa3f7","avatarUrl":"/avatars/723421b31aae5742ae6a64b8e5da66a4.svg","isPro":false,"fullname":"Phuc Nguyen","user":"PhucDucAnhNguyen","type":"user","name":"PhucDucAnhNguyen"},"name":"Phuc D. A. Nguyen","status":"claimed_verified","statusLastChangedAt":"2023-12-19T12:38:47.173Z","hidden":false},{"_id":"658187935cfd2c169a6659fa","name":"Tuan Duc Ngo","hidden":false},{"_id":"658187935cfd2c169a6659fb","name":"Chuang Gan","hidden":false},{"_id":"658187935cfd2c169a6659fc","name":"Evangelos Kalogerakis","hidden":false},{"_id":"658187935cfd2c169a6659fd","user":{"_id":"63e9fc6174f940d171e4c5e9","avatarUrl":"/avatars/0c34fdaecd7bd8aaffb3d68e83c0eeaf.svg","isPro":false,"fullname":"Anh Tran","user":"anhttran1111","type":"user","name":"anhttran1111"},"name":"Anh Tran","status":"claimed_verified","statusLastChangedAt":"2024-12-05T09:30:15.032Z","hidden":false},{"_id":"658187935cfd2c169a6659fe","name":"Cuong Pham","hidden":false},{"_id":"658187935cfd2c169a6659ff","user":{"_id":"63044075d14428368d1c4c6c","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/63044075d14428368d1c4c6c/P2WhQMib5VXyT0w-Sjf2V.jpeg","isPro":false,"fullname":"Khoi Nguyen","user":"ducminhkhoi","type":"user","name":"ducminhkhoi"},"name":"Khoi Nguyen","status":"claimed_verified","statusLastChangedAt":"2024-08-27T07:47:21.966Z","hidden":false}],"publishedAt":"2023-12-17T10:07:03.000Z","title":"Open3DIS: Open-vocabulary 3D Instance Segmentation with 2D Mask Guidance","summary":"We introduce Open3DIS, a novel solution designed to tackle the problem of\nOpen-Vocabulary Instance Segmentation within 3D scenes. Objects within 3D\nenvironments exhibit diverse shapes, scales, and colors, making precise\ninstance-level identification a challenging task. Recent advancements in\nOpen-Vocabulary scene understanding have made significant strides in this area\nby employing class-agnostic 3D instance proposal networks for object\nlocalization and learning queryable features for each 3D mask. While these\nmethods produce high-quality instance proposals, they struggle with identifying\nsmall-scale and geometrically ambiguous objects. The key idea of our method is\na new module that aggregates 2D instance masks across frames and maps them to\ngeometrically coherent point cloud regions as high-quality object proposals\naddressing the above limitations. These are then combined with 3D\nclass-agnostic instance proposals to include a wide range of objects in the\nreal world. To validate our approach, we conducted experiments on three\nprominent datasets, including ScanNet200, S3DIS, and Replica, demonstrating\nsignificant performance gains in segmenting objects with diverse categories\nover the state-of-the-art approaches.","upvotes":1,"discussionId":"658187945cfd2c169a665a4a","githubRepo":"https://github.com/VinAIResearch/Open3DIS","githubRepoAddedBy":"auto","ai_summary":"A novel method, Open3DIS, aggregates 2D instance masks across frames to create high-quality 3D instance proposals, addressing challenges in open-vocabulary instance segmentation in diverse 3D scenes.","ai_keywords":["Open-Vocabulary Instance Segmentation","class-agnostic 3D instance proposal networks","queryable features","point cloud regions","3D class-agnostic instance proposals","ScanNet200","S3DIS","Replica"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":138},"publishedAt":"2023-12-17T05:07:03.000Z","title":"Open3DIS: Open-vocabulary 3D Instance Segmentation with 2D Mask Guidance","summary":"We introduce Open3DIS, a novel solution designed to tackle the problem of\nOpen-Vocabulary Instance Segmentation within 3D scenes. Objects within 3D\nenvironments exhibit diverse shapes, scales, and colors, making precise\ninstance-level identification a challenging task. Recent advancements in\nOpen-Vocabulary scene understanding have made significant strides in this area\nby employing class-agnostic 3D instance proposal networks for object\nlocalization and learning queryable features for each 3D mask. While these\nmethods produce high-quality instance proposals, they struggle with identifying\nsmall-scale and geometrically ambiguous objects. The key idea of our method is\na new module that aggregates 2D instance masks across frames and maps them to\ngeometrically coherent point cloud regions as high-quality object proposals\naddressing the above limitations. These are then combined with 3D\nclass-agnostic instance proposals to include a wide range of objects in the\nreal world. To validate our approach, we conducted experiments on three\nprominent datasets, including ScanNet200, S3DIS, and Replica, demonstrating\nsignificant performance gains in segmenting objects with diverse categories\nover the state-of-the-art approaches.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2312.10671.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Open3DIS: Open-vocabulary 3D Instance Segmentation with 2D Mask Guidance"}],"highlightedSummary":[{"type":"text","text":"We introduce Open3DIS, a novel solution designed to tackle the problem of\nOpen-Vocabulary Instance Segmentation within 3D scenes. Objects within 3D\nenvironments exhibit diverse shapes, scales, and colors, making precise\ninstance-level identification a challenging task. Recent advancements in\nOpen-Vocabulary scene understanding have made significant strides in this area\nby employing class-agnostic 3D instance proposal networks for object\nlocalization and learning queryable features for each 3D mask. While these\nmethods produce high-quality instance proposals, they struggle with identifying\nsmall-scale and geometrically ambiguous objects. The key idea of our method is\na new module that aggregates 2D instance masks across frames and maps them to\ngeometrically coherent point cloud regions as high-quality object proposals\naddressing the above limitations. These are then combined with 3D\nclass-agnostic instance proposals to include a wide range of objects in the\nreal world. To validate our approach, we conducted experiments on three\nprominent datasets, including ScanNet200, S3DIS, and Replica, demonstrating\nsignificant performance gains in segmenting objects with diverse categories\nover the state-of-the-art approaches."}]},{"paper":{"id":"2407.10102","authors":[{"_id":"6731ade57ba655a97ec686b7","name":"Umar Khalid","hidden":false},{"_id":"6731ade57ba655a97ec686b8","name":"Hasan Iqbal","hidden":false},{"_id":"6731ade57ba655a97ec686b9","name":"Azib Farooq","hidden":false},{"_id":"6731ade57ba655a97ec686ba","name":"Jing Hua","hidden":false},{"_id":"6731ade57ba655a97ec686bb","name":"Chen Chen","hidden":false}],"publishedAt":"2024-07-14T07:03:50.000Z","title":"3DEgo: 3D Editing on the Go!","summary":"We introduce 3DEgo to address a novel problem of directly synthesizing\nphotorealistic 3D scenes from monocular videos guided by textual prompts.\nConventional methods construct a text-conditioned 3D scene through a\nthree-stage process, involving pose estimation using Structure-from-Motion\n(SfM) libraries like COLMAP, initializing the 3D model with unedited images,\nand iteratively updating the dataset with edited images to achieve a 3D scene\nwith text fidelity. Our framework streamlines the conventional multi-stage 3D\nediting process into a single-stage workflow by overcoming the reliance on\nCOLMAP and eliminating the cost of model initialization. We apply a diffusion\nmodel to edit video frames prior to 3D scene creation by incorporating our\ndesigned noise blender module for enhancing multi-view editing consistency, a\nstep that does not require additional training or fine-tuning of T2I diffusion\nmodels. 3DEgo utilizes 3D Gaussian Splatting to create 3D scenes from the\nmulti-view consistent edited frames, capitalizing on the inherent temporal\ncontinuity and explicit point cloud data. 3DEgo demonstrates remarkable editing\nprecision, speed, and adaptability across a variety of video sources, as\nvalidated by extensive evaluations on six datasets, including our own prepared\nGS25 dataset. Project Page: https://3dego.github.io/","upvotes":0,"discussionId":"6731adeb7ba655a97ec68812","ai_summary":"3DEgo synthesizes photorealistic 3D scenes from monocular videos using a single-stage diffusion model with 3D Gaussian Splatting, achieving high precision and adaptability.","ai_keywords":["diffusion model","noise blender","3D Gaussian Splatting","photorealistic 3D scenes","monocular videos","textual prompts","Structure-from-Motion","COLMAP","3D scene creation","point cloud data"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-07-14T03:03:50.000Z","title":"3DEgo: 3D Editing on the Go!","summary":"We introduce 3DEgo to address a novel problem of directly synthesizing\nphotorealistic 3D scenes from monocular videos guided by textual prompts.\nConventional methods construct a text-conditioned 3D scene through a\nthree-stage process, involving pose estimation using Structure-from-Motion\n(SfM) libraries like COLMAP, initializing the 3D model with unedited images,\nand iteratively updating the dataset with edited images to achieve a 3D scene\nwith text fidelity. Our framework streamlines the conventional multi-stage 3D\nediting process into a single-stage workflow by overcoming the reliance on\nCOLMAP and eliminating the cost of model initialization. We apply a diffusion\nmodel to edit video frames prior to 3D scene creation by incorporating our\ndesigned noise blender module for enhancing multi-view editing consistency, a\nstep that does not require additional training or fine-tuning of T2I diffusion\nmodels. 3DEgo utilizes 3D Gaussian Splatting to create 3D scenes from the\nmulti-view consistent edited frames, capitalizing on the inherent temporal\ncontinuity and explicit point cloud data. 3DEgo demonstrates remarkable editing\nprecision, speed, and adaptability across a variety of video sources, as\nvalidated by extensive evaluations on six datasets, including our own prepared\nGS25 dataset. Project Page: https://3dego.github.io/","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2407.10102.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"3DEgo: 3D Editing on the Go!"}],"highlightedSummary":[{"type":"highlight","text":"We introduce 3DEgo to address a novel problem of directly synthesizing\nphotorealistic 3D scenes from monocular videos guided by textual prompts."},{"type":"text","text":"\nConventional methods construct a text-conditioned 3D scene through a\nthree-stage process, involving pose estimation using Structure-from-Motion\n(SfM) libraries like COLMAP, initializing the 3D model with unedited images,\nand iteratively updating the dataset with edited images to achieve a 3D scene\nwith text fidelity. Our framework streamlines the conventional multi-stage 3D\nediting process into a single-stage workflow by overcoming the reliance on\nCOLMAP and eliminating the cost of model initialization. We apply a diffusion\nmodel to edit video frames prior to 3D scene creation by incorporating our\ndesigned noise blender module for enhancing multi-view editing consistency, a\nstep that does not require additional training or fine-tuning of T2I diffusion\nmodels. 3DEgo utilizes 3D Gaussian Splatting to create 3D scenes from the\nmulti-view consistent edited frames, capitalizing on the inherent temporal\ncontinuity and explicit point cloud data. 3DEgo demonstrates remarkable editing\nprecision, speed, and adaptability across a variety of video sources, as\nvalidated by extensive evaluations on six datasets, including our own prepared\nGS25 dataset. Project Page: https://3dego.github.io/"}]},{"paper":{"id":"2408.10605","authors":[{"_id":"67074c28679d36bc0e37787a","user":{"_id":"65c09224a9c1b20e69a61569","avatarUrl":"/avatars/78c73be711f2c7a889acb088507ca0aa.svg","isPro":false,"fullname":"YANBO DING","user":"yanboding","type":"user","name":"yanboding"},"name":"Yanbo Ding","status":"claimed_verified","statusLastChangedAt":"2025-05-20T07:23:35.734Z","hidden":false},{"_id":"67074c28679d36bc0e37787b","name":"Shaobin Zhuang","hidden":false},{"_id":"67074c28679d36bc0e37787c","user":{"_id":"61fb81006374891646732f37","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1643872995181-61fb81006374891646732f37.jpeg","isPro":false,"fullname":"Kunchang Li","user":"Andy1621","type":"user","name":"Andy1621"},"name":"Kunchang Li","status":"claimed_verified","statusLastChangedAt":"2025-05-13T10:27:38.451Z","hidden":false},{"_id":"67074c28679d36bc0e37787d","user":{"_id":"66b1b8c6d3407c95f56b7ed2","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/66b1b8c6d3407c95f56b7ed2/R_FV4Y_GvHO2OkRhQqXiL.jpeg","isPro":false,"fullname":"zhengrong yue","user":"yuezhengrong","type":"user","name":"yuezhengrong"},"name":"Zhengrong Yue","status":"claimed_verified","statusLastChangedAt":"2026-05-10T16:57:17.759Z","hidden":false},{"_id":"67074c28679d36bc0e37787e","name":"Yu Qiao","hidden":false},{"_id":"67074c28679d36bc0e37787f","name":"Yali Wang","hidden":false}],"publishedAt":"2024-08-20T07:37:23.000Z","title":"MUSES: 3D-Controllable Image Generation via Multi-Modal Agent\n  Collaboration","summary":"Despite recent advancements in text-to-image generation, most existing\nmethods struggle to create images with multiple objects and complex spatial\nrelationships in 3D world. To tackle this limitation, we introduce a generic AI\nsystem, namely MUSES, for 3D-controllable image generation from user queries.\nSpecifically, our MUSES addresses this challenging task by developing a\nprogressive workflow with three key components, including (1) Layout Manager\nfor 2D-to-3D layout lifting, (2) Model Engineer for 3D object acquisition and\ncalibration, (3) Image Artist for 3D-to-2D image rendering. By mimicking the\ncollaboration of human professionals, this multi-modal agent pipeline\nfacilitates the effective and automatic creation of images with 3D-controllable\nobjects, through an explainable integration of top-down planning and bottom-up\ngeneration. Additionally, we find that existing benchmarks lack detailed\ndescriptions of complex 3D spatial relationships of multiple objects. To fill\nthis gap, we further construct a new benchmark of T2I-3DisBench (3D image\nscene), which describes diverse 3D image scenes with 50 detailed prompts.\nExtensive experiments show the state-of-the-art performance of MUSES on both\nT2I-CompBench and T2I-3DisBench, outperforming recent strong competitors such\nas DALL-E 3 and Stable Diffusion 3. These results demonstrate a significant\nstep of MUSES forward in bridging natural language, 2D image generation, and 3D\nworld. Our codes and models will be released soon.","upvotes":2,"discussionId":"67074c2a679d36bc0e377908","projectPage":"https://github.com/DINGYANB/MUSES","githubRepo":"https://github.com/DINGYANB/MUSES","githubRepoAddedBy":"user","ai_summary":"MUSES is a 3D-controllable image generation system that uses a progressive workflow with components for layout lifting, object acquisition, and image rendering, outperforming existing methods on both 2D and 3D tasks.","ai_keywords":["Layout Manager","Model Engineer","Image Artist","multi-modal agent pipeline","top-down planning","bottom-up generation","T2I-3DisBench","DALL-E 3","Stable Diffusion 3"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":38},"publishedAt":"2024-08-20T03:37:23.000Z","title":"MUSES: 3D-Controllable Image Generation via Multi-Modal Agent\n  Collaboration","summary":"Despite recent advancements in text-to-image generation, most existing\nmethods struggle to create images with multiple objects and complex spatial\nrelationships in 3D world. To tackle this limitation, we introduce a generic AI\nsystem, namely MUSES, for 3D-controllable image generation from user queries.\nSpecifically, our MUSES addresses this challenging task by developing a\nprogressive workflow with three key components, including (1) Layout Manager\nfor 2D-to-3D layout lifting, (2) Model Engineer for 3D object acquisition and\ncalibration, (3) Image Artist for 3D-to-2D image rendering. By mimicking the\ncollaboration of human professionals, this multi-modal agent pipeline\nfacilitates the effective and automatic creation of images with 3D-controllable\nobjects, through an explainable integration of top-down planning and bottom-up\ngeneration. Additionally, we find that existing benchmarks lack detailed\ndescriptions of complex 3D spatial relationships of multiple objects. To fill\nthis gap, we further construct a new benchmark of T2I-3DisBench (3D image\nscene), which describes diverse 3D image scenes with 50 detailed prompts.\nExtensive experiments show the state-of-the-art performance of MUSES on both\nT2I-CompBench and T2I-3DisBench, outperforming recent strong competitors such\nas DALL-E 3 and Stable Diffusion 3. These results demonstrate a significant\nstep of MUSES forward in bridging natural language, 2D image generation, and 3D\nworld. Our codes and models will be released soon.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2408.10605.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"MUSES: 3D-Controllable Image Generation via Multi-Modal Agent\n  Collaboration"}],"highlightedSummary":[{"type":"text","text":"Despite recent advancements in text-to-image generation, most existing\nmethods struggle to create images with multiple objects and complex spatial\nrelationships in 3D world. To tackle this limitation, we introduce a generic AI\nsystem, namely MUSES, for 3D-controllable image generation from user queries.\nSpecifically, our MUSES addresses this challenging task by developing a\nprogressive workflow with three key components, including (1) Layout Manager\nfor 2D-to-3D layout lifting, (2) Model Engineer for 3D object acquisition and\ncalibration, (3) Image Artist for 3D-to-2D image rendering. By mimicking the\ncollaboration of human professionals, this multi-modal agent pipeline\nfacilitates the effective and automatic creation of images with 3D-controllable\nobjects, through an explainable integration of top-down planning and bottom-up\ngeneration. Additionally, we find that existing benchmarks lack detailed\ndescriptions of complex 3D spatial relationships of multiple objects. "},{"type":"highlight","text":"To fill\nthis gap, we further construct a new benchmark of T2I-3DisBench (3D image\nscene), which describes diverse 3D image scenes with 50 detailed prompts."},{"type":"text","text":"\nExtensive experiments show the state-of-the-art performance of MUSES on both\nT2I-CompBench and T2I-3DisBench, outperforming recent strong competitors such\nas DALL-E 3 and Stable Diffusion 3. These results demonstrate a significant\nstep of MUSES forward in bridging natural language, 2D image generation, and 3D\nworld. Our codes and models will be released soon."}]},{"paper":{"id":"2606.31924","authors":[{"_id":"6a44bcbb41f04ae4d7ad99a0","user":{"_id":"633e848d26bad5c614000fbf","avatarUrl":"/avatars/fdb0ed55973613fbd0e8c7e75db4ac59.svg","isPro":false,"fullname":"liuxiaoyu","user":"xiaoyu1104","type":"user","name":"xiaoyu1104"},"name":"Xiaoyu Liu","status":"claimed_verified","statusLastChangedAt":"2026-07-01T08:44:07.203Z","hidden":false},{"_id":"6a44bcbb41f04ae4d7ad99a1","name":"Huan Wang","hidden":false},{"_id":"6a44bcbb41f04ae4d7ad99a2","name":"Fan Li","hidden":false},{"_id":"6a44bcbb41f04ae4d7ad99a3","name":"Zhixin Wang","hidden":false},{"_id":"6a44bcbb41f04ae4d7ad99a4","name":"Jiaqi Xu","hidden":false},{"_id":"6a44bcbb41f04ae4d7ad99a5","name":"Ming Liu","hidden":false},{"_id":"6a44bcbb41f04ae4d7ad99a6","name":"Wangmeng Zuo","hidden":false}],"publishedAt":"2026-06-30T00:00:00.000Z","title":"InstanceControl: Controllable Complex Image Generation without Instance Labeling","summary":"Controllable image generation methods, such as ControlNet, have demonstrated a remarkable capacity to introduce visual conditions(e.g., depth maps) to guide image generation. However, these methods often struggle with complex multi-instance scenes, frequently leading to attribute confusion among instances. While recent approaches attempt to mitigate this via manual instance labeling, such requirements are labor-intensive. In this paper, we propose InstanceControl, a novel multi-instance controllable generation method that eliminates the need for instance labeling. We identify the primary bottleneck in existing methods as the inability to accurately associate instance descriptions with their corresponding regions within visual conditions. To address this, we leverage the Vision-Language Model (VLM) to establish instance-level correspondences between text prompts and visual conditions. Specifically, the VLM automatically parses instance descriptions from the text prompts and simultaneously predicts instance masks based on the visual conditions. Furthermore, since the predicted masks may contain noise, we introduce an adaptive mask refinement strategy that dynamically refines these instance masks during the generation process. Extensive experiments demonstrate that our approach outperforms state-of-the-art methods, achieving superior fidelity and precise instance-level control.","upvotes":11,"discussionId":"6a44bcbc41f04ae4d7ad99a7","projectPage":"https://instancecontrol.github.io/InstanceControl/","githubRepo":"https://github.com/liuxiaoyu1104/InstanceControl","githubRepoAddedBy":"user","ai_summary":"InstanceControl enables multi-instance image generation by using vision-language models to establish instance-level correspondences between text prompts and visual conditions, while employing adaptive mask refinement for improved accuracy.","ai_keywords":["ControlNet","Vision-Language Model","instance-level correspondences","instance masks","adaptive mask refinement","multi-instance scenes","visual conditions","text prompts"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":21},"publishedAt":"2026-06-29T20:00:00.000Z","title":"InstanceControl: Controllable Complex Image Generation without Instance Labeling","summary":"Controllable image generation methods, such as ControlNet, have demonstrated a remarkable capacity to introduce visual conditions(e.g., depth maps) to guide image generation. However, these methods often struggle with complex multi-instance scenes, frequently leading to attribute confusion among instances. While recent approaches attempt to mitigate this via manual instance labeling, such requirements are labor-intensive. In this paper, we propose InstanceControl, a novel multi-instance controllable generation method that eliminates the need for instance labeling. We identify the primary bottleneck in existing methods as the inability to accurately associate instance descriptions with their corresponding regions within visual conditions. To address this, we leverage the Vision-Language Model (VLM) to establish instance-level correspondences between text prompts and visual conditions. Specifically, the VLM automatically parses instance descriptions from the text prompts and simultaneously predicts instance masks based on the visual conditions. Furthermore, since the predicted masks may contain noise, we introduce an adaptive mask refinement strategy that dynamically refines these instance masks during the generation process. Extensive experiments demonstrate that our approach outperforms state-of-the-art methods, achieving superior fidelity and precise instance-level control.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2606.31924.png","numComments":4,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"InstanceControl: Controllable Complex Image Generation without Instance Labeling"}],"highlightedSummary":[{"type":"text","text":"Controllable image generation methods, such as ControlNet, have demonstrated a remarkable capacity to introduce visual conditions(e.g., depth maps) to guide image generation. "},{"type":"highlight","text":"However, these methods often struggle with complex multi-instance scenes, frequently leading to attribute confusion among instances."},{"type":"text","text":" While recent approaches attempt to mitigate this via manual instance labeling, such requirements are labor-intensive. In this paper, we propose InstanceControl, a novel multi-instance controllable generation method that eliminates the need for instance labeling. We identify the primary bottleneck in existing methods as the inability to accurately associate instance descriptions with their corresponding regions within visual conditions. To address this, we leverage the Vision-Language Model (VLM) to establish instance-level correspondences between text prompts and visual conditions. Specifically, the VLM automatically parses instance descriptions from the text prompts and simultaneously predicts instance masks based on the visual conditions. Furthermore, since the predicted masks may contain noise, we introduce an adaptive mask refinement strategy that dynamically refines these instance masks during the generation process. Extensive experiments demonstrate that our approach outperforms state-of-the-art methods, achieving superior fidelity and precise instance-level control."}]},{"paper":{"id":"2310.17994","authors":[{"_id":"653f2185b424289c5f839a0d","user":{"_id":"6451a5d7938967fd069fd135","avatarUrl":"/avatars/56a00b2e6211f678ec040ea379e7387a.svg","isPro":false,"fullname":"Kyle Sargent","user":"ksahug","type":"user","name":"ksahug"},"name":"Kyle Sargent","status":"admin_assigned","statusLastChangedAt":"2023-10-30T13:37:25.459Z","hidden":false},{"_id":"653f2185b424289c5f839a0e","name":"Zizhang Li","hidden":false},{"_id":"653f2185b424289c5f839a0f","name":"Tanmay Shah","hidden":false},{"_id":"653f2185b424289c5f839a10","user":{"_id":"64c7f088c864d962eddeaf8c","avatarUrl":"/avatars/e6364c56e6cb6907b6dd2d34eed83720.svg","isPro":false,"fullname":"Charles Herrmann","user":"irwinherrmann","type":"user","name":"irwinherrmann"},"name":"Charles Herrmann","status":"admin_assigned","statusLastChangedAt":"2023-10-30T13:38:03.772Z","hidden":false},{"_id":"653f2185b424289c5f839a11","name":"Hong-Xing Yu","hidden":false},{"_id":"653f2185b424289c5f839a12","user":{"_id":"63793aee2cb50dda39d6e73b","avatarUrl":"/avatars/1d7eb1a19ce2e18802e9481200eb03f7.svg","isPro":false,"fullname":"Yunzhi Zhang","user":"Yunzhi","type":"user","name":"Yunzhi"},"name":"Yunzhi Zhang","status":"admin_assigned","statusLastChangedAt":"2023-10-30T13:38:31.287Z","hidden":false},{"_id":"653f2185b424289c5f839a13","user":{"_id":"6338dd0db7ce9192552cdf7f","avatarUrl":"/avatars/d0080d797490926c53690c1b7602d232.svg","isPro":false,"fullname":"Eric Chan","user":"ericryanchan","type":"user","name":"ericryanchan"},"name":"Eric Ryan Chan","status":"admin_assigned","statusLastChangedAt":"2023-10-30T13:38:41.553Z","hidden":false},{"_id":"653f2185b424289c5f839a14","name":"Dmitry Lagun","hidden":false},{"_id":"653f2185b424289c5f839a15","name":"Li Fei-Fei","hidden":false},{"_id":"653f2185b424289c5f839a16","name":"Deqing Sun","hidden":false},{"_id":"653f2185b424289c5f839a17","user":{"_id":"647b7b8345616c3de3627863","avatarUrl":"/avatars/8196364bffed71086ba9a0a4a62027d3.svg","isPro":false,"fullname":"Jiajun Wu","user":"jiajunwu","type":"user","name":"jiajunwu"},"name":"Jiajun Wu","status":"claimed_verified","statusLastChangedAt":"2023-12-21T09:32:01.925Z","hidden":false}],"publishedAt":"2023-10-27T09:06:43.000Z","title":"ZeroNVS: Zero-Shot 360-Degree View Synthesis from a Single Real Image","summary":"We introduce a 3D-aware diffusion model, ZeroNVS, for single-image novel view\nsynthesis for in-the-wild scenes. While existing methods are designed for\nsingle objects with masked backgrounds, we propose new techniques to address\nchallenges introduced by in-the-wild multi-object scenes with complex\nbackgrounds. Specifically, we train a generative prior on a mixture of data\nsources that capture object-centric, indoor, and outdoor scenes. To address\nissues from data mixture such as depth-scale ambiguity, we propose a novel\ncamera conditioning parameterization and normalization scheme. Further, we\nobserve that Score Distillation Sampling (SDS) tends to truncate the\ndistribution of complex backgrounds during distillation of 360-degree scenes,\nand propose \"SDS anchoring\" to improve the diversity of synthesized novel\nviews. Our model sets a new state-of-the-art result in LPIPS on the DTU dataset\nin the zero-shot setting, even outperforming methods specifically trained on\nDTU. We further adapt the challenging Mip-NeRF 360 dataset as a new benchmark\nfor single-image novel view synthesis, and demonstrate strong performance in\nthis setting. Our code and data are at http://kylesargent.github.io/zeronvs/","upvotes":8,"discussionId":"653f218ab424289c5f839a92","githubRepo":"https://github.com/kylesargent/zeronvs","githubRepoAddedBy":"auto","ai_summary":"A 3D-aware diffusion model, ZeroNVS, addresses challenges in single-image novel view synthesis for complex, in-the-wild scenes, setting new benchmarks and improving diversity in synthesized views.","ai_keywords":["3D-aware diffusion model","novel view synthesis","data mixture","depth-scale ambiguity","camera conditioning","normalization scheme","Score Distillation Sampling","SDS anchoring","LPIPS","DTU dataset","Mip-NeRF 360 dataset"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":535},"publishedAt":"2023-10-27T05:06:43.000Z","title":"ZeroNVS: Zero-Shot 360-Degree View Synthesis from a Single Real Image","summary":"We introduce a 3D-aware diffusion model, ZeroNVS, for single-image novel view\nsynthesis for in-the-wild scenes. While existing methods are designed for\nsingle objects with masked backgrounds, we propose new techniques to address\nchallenges introduced by in-the-wild multi-object scenes with complex\nbackgrounds. Specifically, we train a generative prior on a mixture of data\nsources that capture object-centric, indoor, and outdoor scenes. To address\nissues from data mixture such as depth-scale ambiguity, we propose a novel\ncamera conditioning parameterization and normalization scheme. Further, we\nobserve that Score Distillation Sampling (SDS) tends to truncate the\ndistribution of complex backgrounds during distillation of 360-degree scenes,\nand propose \"SDS anchoring\" to improve the diversity of synthesized novel\nviews. Our model sets a new state-of-the-art result in LPIPS on the DTU dataset\nin the zero-shot setting, even outperforming methods specifically trained on\nDTU. We further adapt the challenging Mip-NeRF 360 dataset as a new benchmark\nfor single-image novel view synthesis, and demonstrate strong performance in\nthis setting. Our code and data are at http://kylesargent.github.io/zeronvs/","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2310.17994.png","numComments":1,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"ZeroNVS: Zero-Shot 360-Degree View Synthesis from a Single Real Image"}],"highlightedSummary":[{"type":"text","text":"We introduce a 3D-aware diffusion model, ZeroNVS, for single-image novel view\nsynthesis for in-the-wild scenes. "},{"type":"highlight","text":"While existing methods are designed for\nsingle objects with masked backgrounds, we propose new techniques to address\nchallenges introduced by in-the-wild multi-object scenes with complex\nbackgrounds."},{"type":"text","text":" Specifically, we train a generative prior on a mixture of data\nsources that capture object-centric, indoor, and outdoor scenes. To address\nissues from data mixture such as depth-scale ambiguity, we propose a novel\ncamera conditioning parameterization and normalization scheme. Further, we\nobserve that Score Distillation Sampling (SDS) tends to truncate the\ndistribution of complex backgrounds during distillation of 360-degree scenes,\nand propose \"SDS anchoring\" to improve the diversity of synthesized novel\nviews. Our model sets a new state-of-the-art result in LPIPS on the DTU dataset\nin the zero-shot setting, even outperforming methods specifically trained on\nDTU. We further adapt the challenging Mip-NeRF 360 dataset as a new benchmark\nfor single-image novel view synthesis, and demonstrate strong performance in\nthis setting. Our code and data are at http://kylesargent.github.io/zeronvs/"}]},{"paper":{"id":"2307.11073","authors":[{"_id":"64c0c745a8a2dcaa167ee02c","name":"Oscar Michel","hidden":false},{"_id":"64c0c745a8a2dcaa167ee02d","user":{"_id":"6303d5924ec2dfa82a5627de","avatarUrl":"/avatars/1e433b7fb9d30b9f42dec487d658b1e3.svg","isPro":false,"fullname":"Anand Bhattad","user":"anandbhattad","type":"user","name":"anandbhattad"},"name":"Anand Bhattad","status":"claimed_verified","statusLastChangedAt":"2025-06-29T10:15:44.515Z","hidden":false},{"_id":"64c0c745a8a2dcaa167ee02e","name":"Eli VanderBilt","hidden":false},{"_id":"64c0c745a8a2dcaa167ee02f","name":"Ranjay Krishna","hidden":false},{"_id":"64c0c745a8a2dcaa167ee030","name":"Aniruddha Kembhavi","hidden":false},{"_id":"64c0c745a8a2dcaa167ee031","user":{"_id":"62a108242e30aaf94ec283bb","avatarUrl":"/avatars/4da19ae99df6a3fa2d81336eb59cdaa7.svg","isPro":false,"fullname":"Tanmay Gupta","user":"tanmayg","type":"user","name":"tanmayg"},"name":"Tanmay Gupta","status":"claimed_verified","statusLastChangedAt":"2025-06-10T09:33:07.680Z","hidden":false}],"publishedAt":"2023-07-20T17:53:46.000Z","title":"OBJECT 3DIT: Language-guided 3D-aware Image Editing","summary":"Existing image editing tools, while powerful, typically disregard the\nunderlying 3D geometry from which the image is projected. As a result, edits\nmade using these tools may become detached from the geometry and lighting\nconditions that are at the foundation of the image formation process. In this\nwork, we formulate the newt ask of language-guided 3D-aware editing, where\nobjects in an image should be edited according to a language instruction in\ncontext of the underlying 3D scene. To promote progress towards this goal, we\nrelease OBJECT: a dataset consisting of 400K editing examples created from\nprocedurally generated 3D scenes. Each example consists of an input image,\nediting instruction in language, and the edited image. We also introduce 3DIT :\nsingle and multi-task models for four editing tasks. Our models show impressive\nabilities to understand the 3D composition of entire scenes, factoring in\nsurrounding objects, surfaces, lighting conditions, shadows, and\nphysically-plausible object configurations. Surprisingly, training on only\nsynthetic scenes from OBJECT, editing capabilities of 3DIT generalize to\nreal-world images.","upvotes":0,"discussionId":"64c0c746a8a2dcaa167ee050","githubRepo":"https://github.com/allenai/object-edit","githubRepoAddedBy":"auto","ai_summary":"A new task of language-guided 3D-aware image editing is introduced, along with a dataset and models that can understand and edit based on 3D scene compositions.","ai_keywords":["3D-aware editing","OBJECT","3DIT","procedurally generated 3D scenes","editing instruction","input image","edited image","single and multi-task models","3D composition","surrounding objects","surfaces","lighting conditions","shadows","physically-plausible object configurations"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":29},"publishedAt":"2023-07-20T13:53:46.000Z","title":"OBJECT 3DIT: Language-guided 3D-aware Image Editing","summary":"Existing image editing tools, while powerful, typically disregard the\nunderlying 3D geometry from which the image is projected. As a result, edits\nmade using these tools may become detached from the geometry and lighting\nconditions that are at the foundation of the image formation process. In this\nwork, we formulate the newt ask of language-guided 3D-aware editing, where\nobjects in an image should be edited according to a language instruction in\ncontext of the underlying 3D scene. To promote progress towards this goal, we\nrelease OBJECT: a dataset consisting of 400K editing examples created from\nprocedurally generated 3D scenes. Each example consists of an input image,\nediting instruction in language, and the edited image. We also introduce 3DIT :\nsingle and multi-task models for four editing tasks. Our models show impressive\nabilities to understand the 3D composition of entire scenes, factoring in\nsurrounding objects, surfaces, lighting conditions, shadows, and\nphysically-plausible object configurations. Surprisingly, training on only\nsynthetic scenes from OBJECT, editing capabilities of 3DIT generalize to\nreal-world images.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2307.11073.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"OBJECT 3DIT: Language-guided 3D-aware Image Editing"}],"highlightedSummary":[{"type":"text","text":"Existing image editing tools, while powerful, typically disregard the\nunderlying 3D geometry from which the image is projected. As a result, edits\nmade using these tools may become detached from the geometry and lighting\nconditions that are at the foundation of the image formation process. In this\nwork, we formulate the newt ask of language-guided 3D-aware editing, where\nobjects in an image should be edited according to a language instruction in\ncontext of the underlying 3D scene. To promote progress towards this goal, we\nrelease OBJECT: a dataset consisting of 400K editing examples created from\nprocedurally generated 3D scenes. Each example consists of an input image,\nediting instruction in language, and the edited image. We also introduce 3DIT :\nsingle and multi-task models for four editing tasks. "},{"type":"highlight","text":"Our models show impressive\nabilities to understand the 3D composition of entire scenes, factoring in\nsurrounding objects, surfaces, lighting conditions, shadows, and\nphysically-plausible object configurations."},{"type":"text","text":" Surprisingly, training on only\nsynthetic scenes from OBJECT, editing capabilities of 3DIT generalize to\nreal-world images."}]},{"paper":{"id":"2602.06166","authors":[{"_id":"6a06d52db1a8cbabc9f09b86","name":"Bangji Yang","hidden":false},{"_id":"6a06d52db1a8cbabc9f09b87","name":"Ruihan Guo","hidden":false},{"_id":"6a06d52db1a8cbabc9f09b88","name":"Jiajun Fan","hidden":false},{"_id":"6a06d52db1a8cbabc9f09b89","name":"Chaoran Cheng","hidden":false},{"_id":"6a06d52db1a8cbabc9f09b8a","name":"Ge Liu","hidden":false}],"publishedAt":"2026-02-05T00:00:00.000Z","title":"M3: High-fidelity Text-to-Image Generation via Multi-Modal, Multi-Agent and Multi-Round Visual Reasoning","summary":"Generative models have achieved impressive fidelity in text-to-image synthesis, yet struggle with complex compositional prompts involving multiple constraints. We introduce M3 (Multi-Modal, Multi-Agent, Multi-Round), a training-free framework that systematically resolves these failures through iterative inference-time refinement. M3 orchestrates off-the-shelf foundation models in a robust multi-agent loop: a Planner decomposes prompts into verifiable checklists, while specialized Checker, Refiner, and Editor agents surgically correct constraints one at a time, with a Verifier ensuring monotonic improvement. Applied to open-source models, M3 achieves remarkable results on the challenging OneIG-EN benchmark, with our Qwen-Image+M3 surpassing commercial flagship systems including Imagen4 (0.515) and Seedream 3.0 (0.530), reaching state-of-the-art performance (0.532 overall). This demonstrates that intelligent multi-agent reasoning can elevate open-source models beyond proprietary alternatives. M3 also substantially improves GenEval compositional metrics, effectively doubling spatial reasoning performance on hardened test sets. As a plug-and-play module compatible with any pre-trained T2I model, M3 establishes a new paradigm for compositional generation without costly retraining.","upvotes":0,"discussionId":"6a06d52db1a8cbabc9f09b8b","ai_summary":"M3 is a training-free, multi-agent framework that enhances text-to-image generation by iteratively refining complex compositional prompts through specialized agents working in concert.","ai_keywords":["text-to-image synthesis","multi-agent system","iterative inference-time refinement","prompt decomposition","constraint correction","compositional generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2026-02-04T19:00:00.000Z","title":"M3: High-fidelity Text-to-Image Generation via Multi-Modal, Multi-Agent and Multi-Round Visual Reasoning","summary":"Generative models have achieved impressive fidelity in text-to-image synthesis, yet struggle with complex compositional prompts involving multiple constraints. We introduce M3 (Multi-Modal, Multi-Agent, Multi-Round), a training-free framework that systematically resolves these failures through iterative inference-time refinement. M3 orchestrates off-the-shelf foundation models in a robust multi-agent loop: a Planner decomposes prompts into verifiable checklists, while specialized Checker, Refiner, and Editor agents surgically correct constraints one at a time, with a Verifier ensuring monotonic improvement. Applied to open-source models, M3 achieves remarkable results on the challenging OneIG-EN benchmark, with our Qwen-Image+M3 surpassing commercial flagship systems including Imagen4 (0.515) and Seedream 3.0 (0.530), reaching state-of-the-art performance (0.532 overall). This demonstrates that intelligent multi-agent reasoning can elevate open-source models beyond proprietary alternatives. M3 also substantially improves GenEval compositional metrics, effectively doubling spatial reasoning performance on hardened test sets. As a plug-and-play module compatible with any pre-trained T2I model, M3 establishes a new paradigm for compositional generation without costly retraining.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2602.06166.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"M3: High-fidelity Text-to-Image Generation via Multi-Modal, Multi-Agent and Multi-Round Visual Reasoning"}],"highlightedSummary":[{"type":"highlight","text":"Generative models have achieved impressive fidelity in text-to-image synthesis, yet struggle with complex compositional prompts involving multiple constraints."},{"type":"text","text":" We introduce M3 (Multi-Modal, Multi-Agent, Multi-Round), a training-free framework that systematically resolves these failures through iterative inference-time refinement. M3 orchestrates off-the-shelf foundation models in a robust multi-agent loop: a Planner decomposes prompts into verifiable checklists, while specialized Checker, Refiner, and Editor agents surgically correct constraints one at a time, with a Verifier ensuring monotonic improvement. Applied to open-source models, M3 achieves remarkable results on the challenging OneIG-EN benchmark, with our Qwen-Image+M3 surpassing commercial flagship systems including Imagen4 (0.515) and Seedream 3.0 (0.530), reaching state-of-the-art performance (0.532 overall). This demonstrates that intelligent multi-agent reasoning can elevate open-source models beyond proprietary alternatives. M3 also substantially improves GenEval compositional metrics, effectively doubling spatial reasoning performance on hardened test sets. As a plug-and-play module compatible with any pre-trained T2I model, M3 establishes a new paradigm for compositional generation without costly retraining."}]},{"paper":{"id":"2603.06873","authors":[{"_id":"69b0f32f1f7902d47f6dfa84","name":"Hang Zhou","hidden":false},{"_id":"69b0f32f1f7902d47f6dfa85","name":"Xinxin Zuo","hidden":false},{"_id":"69b0f32f1f7902d47f6dfa86","name":"Sen Wang","hidden":false},{"_id":"69b0f32f1f7902d47f6dfa87","name":"Li Cheng","hidden":false}],"publishedAt":"2026-03-06T20:48:21.000Z","title":"PICS: Pairwise Image Compositing with Spatial Interactions","summary":"Despite strong single-turn performance, diffusion-based image compositing often struggles to preserve coherent spatial relations in pairwise or sequential edits, where subsequent insertions may overwrite previously generated content and disrupt physical consistency. We introduce PICS, a self-supervised composition-by-decomposition paradigm that composes objects in parallel while explicitly modeling the compositional interactions among (fully-/partially-)visible objects and background. At its core, an Interaction Transformer employs mask-guided Mixture-of-Experts to route background, exclusive, and overlap regions to dedicated experts, with an adaptive α-blending strategy that infers a compatibility-aware fusion of overlapping objects while preserving boundary fidelity. To further enhance robustness to geometric variations, we incorporate geometry-aware augmentations covering both out-of-plane and in-plane pose changes of objects. Our method delivers superior pairwise compositing quality and substantially improved stability, with extensive evaluations across virtual try-on, indoor, and street scene settings showing consistent gains over state-of-the-art baselines. Code and data are available at https://github.com/RyanHangZhou/PICS","upvotes":0,"discussionId":"69b0f32f1f7902d47f6dfa88","ai_summary":"PICS introduces a self-supervised parallel composition paradigm for diffusion-based image compositing that models compositional interactions and improves spatial coherence through interaction transformers and adaptive blending.","ai_keywords":["diffusion-based image compositing","self-supervised composition-by-decomposition","Interaction Transformer","mask-guided Mixture-of-Experts","α-blending strategy","geometry-aware augmentations","out-of-plane pose changes","in-plane pose changes"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2026-03-06T15:48:21.000Z","title":"PICS: Pairwise Image Compositing with Spatial Interactions","summary":"Despite strong single-turn performance, diffusion-based image compositing often struggles to preserve coherent spatial relations in pairwise or sequential edits, where subsequent insertions may overwrite previously generated content and disrupt physical consistency. We introduce PICS, a self-supervised composition-by-decomposition paradigm that composes objects in parallel while explicitly modeling the compositional interactions among (fully-/partially-)visible objects and background. At its core, an Interaction Transformer employs mask-guided Mixture-of-Experts to route background, exclusive, and overlap regions to dedicated experts, with an adaptive α-blending strategy that infers a compatibility-aware fusion of overlapping objects while preserving boundary fidelity. To further enhance robustness to geometric variations, we incorporate geometry-aware augmentations covering both out-of-plane and in-plane pose changes of objects. Our method delivers superior pairwise compositing quality and substantially improved stability, with extensive evaluations across virtual try-on, indoor, and street scene settings showing consistent gains over state-of-the-art baselines. Code and data are available at https://github.com/RyanHangZhou/PICS","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2603.06873.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"PICS: Pairwise Image Compositing with Spatial Interactions"}],"highlightedSummary":[{"type":"text","text":"Despite strong single-turn performance, diffusion-based image compositing often struggles to preserve coherent spatial relations in pairwise or sequential edits, where subsequent insertions may overwrite previously generated content and disrupt physical consistency. "},{"type":"highlight","text":"We introduce PICS, a self-supervised composition-by-decomposition paradigm that composes objects in parallel while explicitly modeling the compositional interactions among (fully-/partially-)visible objects and background."},{"type":"text","text":" At its core, an Interaction Transformer employs mask-guided Mixture-of-Experts to route background, exclusive, and overlap regions to dedicated experts, with an adaptive α-blending strategy that infers a compatibility-aware fusion of overlapping objects while preserving boundary fidelity. To further enhance robustness to geometric variations, we incorporate geometry-aware augmentations covering both out-of-plane and in-plane pose changes of objects. Our method delivers superior pairwise compositing quality and substantially improved stability, with extensive evaluations across virtual try-on, indoor, and street scene settings showing consistent gains over state-of-the-art baselines. Code and data are available at https://github.com/RyanHangZhou/PICS"}]},{"paper":{"id":"2609.23796","authors":[{"_id":"6ab2275938788866c1a07066","name":"Yang-Tian Sun","hidden":false},{"_id":"6ab2275938788866c1a07067","name":"Tianjia Liu","hidden":false},{"_id":"6ab2275938788866c1a07068","name":"Zehuan Huang","hidden":false},{"_id":"6ab2275938788866c1a07069","name":"Yi-Hua Huang","hidden":false},{"_id":"6ab2275938788866c1a0706a","name":"Xiaoyang Lyu","hidden":false},{"_id":"6ab2275938788866c1a0706b","name":"Ziyi Yang","hidden":false},{"_id":"6ab2275938788866c1a0706c","name":"Zi-Xin Zou","hidden":false},{"_id":"6ab2275938788866c1a0706d","name":"Yuan-Chen Guo","hidden":false},{"_id":"6ab2275938788866c1a0706e","name":"Yan-Pei Cao","hidden":false},{"_id":"6ab2275938788866c1a0706f","name":"Xiaojuan Qi","hidden":false}],"publishedAt":"2026-09-20T00:00:00.000Z","title":"Mira-Scene: Pixel-Aligned Layouts for Generative 3D Scene","summary":"Single-image 3D object generation can now produce high-fidelity assets, yet accurately placing them into a coherent scene layout remains an open challenge. A central difficulty lies in how object layout is represented. Holistic methods absorb placement into a scene-level generation process, sacrificing object-level detail. Compositional methods preserve object fidelity by decoupling geometry from layout, but typically parameterize layout as sparse, unbounded pose variables that are difficult to learn and generalize poorly under scarce scene-level supervision.We present Mira-Scene, a compositional 3D scene reconstruction framework that replaces sparse pose regression with dense, bounded correspondence recovery. At its core is the Canonical Coordinate Map (CCM), a pixel-aligned field that maps each visible object pixel to a surface coordinate in the object's bounded canonical space. When paired with a scene-space Point Cloud Map (PCM) from monocular geometry estimation, CCM induces dense canonical-to-scene correspondences from which object transformations are recovered through robust geometric alignment. Because CCM operates in bounded canonical space, it provides a stable prediction target that can be trained from scalable object-level 3D data without requiring scene-level layout annotations. Mira-Scene further introduces a multimodal diffusion transformer that jointly generates object geometry and CCMs, using modality-specific expert streams with shared attention and positional encoding to promote geometry-layout consistency. Experiments on indoor, outdoor, synthetic, and in-the-wild scenes show that Mira-Scene substantially outperforms strong baselines in layout accuracy, achieving relative gains of 39.8% in 3D-IoU and 16.5% in 2D-IoU over SAM3D, using limited open-source training data.","upvotes":5,"discussionId":"6ab2275938788866c1a07070","projectPage":"https://sunyangtian.github.io/Mira-Scene-web/","githubRepo":"https://github.com/VAST-AI-Research/Mira-Scene","githubRepoAddedBy":"user","githubStars":177,"organization":{"_id":"66deb312fd7d68a29348aa8d","name":"TheHKU","fullname":"Hong Kong University","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/66dc525add44163a31059cf6/kyqlTADY27mPRTqznqQFL.png"}},"publishedAt":"2026-09-19T20:00:00.000Z","title":"Mira-Scene: Pixel-Aligned Layouts for Generative 3D Scene","summary":"Single-image 3D object generation can now produce high-fidelity assets, yet accurately placing them into a coherent scene layout remains an open challenge. A central difficulty lies in how object layout is represented. Holistic methods absorb placement into a scene-level generation process, sacrificing object-level detail. Compositional methods preserve object fidelity by decoupling geometry from layout, but typically parameterize layout as sparse, unbounded pose variables that are difficult to learn and generalize poorly under scarce scene-level supervision.We present Mira-Scene, a compositional 3D scene reconstruction framework that replaces sparse pose regression with dense, bounded correspondence recovery. At its core is the Canonical Coordinate Map (CCM), a pixel-aligned field that maps each visible object pixel to a surface coordinate in the object's bounded canonical space. When paired with a scene-space Point Cloud Map (PCM) from monocular geometry estimation, CCM induces dense canonical-to-scene correspondences from which object transformations are recovered through robust geometric alignment. Because CCM operates in bounded canonical space, it provides a stable prediction target that can be trained from scalable object-level 3D data without requiring scene-level layout annotations. Mira-Scene further introduces a multimodal diffusion transformer that jointly generates object geometry and CCMs, using modality-specific expert streams with shared attention and positional encoding to promote geometry-layout consistency. Experiments on indoor, outdoor, synthetic, and in-the-wild scenes show that Mira-Scene substantially outperforms strong baselines in layout accuracy, achieving relative gains of 39.8% in 3D-IoU and 16.5% in 2D-IoU over SAM3D, using limited open-source training data.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2609.23796.png","numComments":2,"upvoted":false,"organization":{"_id":"66deb312fd7d68a29348aa8d","name":"TheHKU","fullname":"Hong Kong University","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/66dc525add44163a31059cf6/kyqlTADY27mPRTqznqQFL.png"},"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Mira-Scene: Pixel-Aligned Layouts for Generative 3D Scene"}],"highlightedSummary":[{"type":"highlight","text":"Single-image 3D object generation can now produce high-fidelity assets, yet accurately placing them into a coherent scene layout remains an open challenge."},{"type":"text","text":" A central difficulty lies in how object layout is represented. Holistic methods absorb placement into a scene-level generation process, sacrificing object-level detail. Compositional methods preserve object fidelity by decoupling geometry from layout, but typically parameterize layout as sparse, unbounded pose variables that are difficult to learn and generalize poorly under scarce scene-level supervision.We present Mira-Scene, a compositional 3D scene reconstruction framework that replaces sparse pose regression with dense, bounded correspondence recovery. At its core is the Canonical Coordinate Map (CCM), a pixel-aligned field that maps each visible object pixel to a surface coordinate in the object's bounded canonical space. When paired with a scene-space Point Cloud Map (PCM) from monocular geometry estimation, CCM induces dense canonical-to-scene correspondences from which object transformations are recovered through robust geometric alignment. Because CCM operates in bounded canonical space, it provides a stable prediction target that can be trained from scalable object-level 3D data without requiring scene-level layout annotations. Mira-Scene further introduces a multimodal diffusion transformer that jointly generates object geometry and CCMs, using modality-specific expert streams with shared attention and positional encoding to promote geometry-layout consistency. Experiments on indoor, outdoor, synthetic, and in-the-wild scenes show that Mira-Scene substantially outperforms strong baselines in layout accuracy, achieving relative gains of 39.8% in 3D-IoU and 16.5% in 2D-IoU over SAM3D, using limited open-source training data."}]},{"paper":{"id":"2312.00093","authors":[{"_id":"656d30f1f7be0986b4b5a351","user":{"_id":"6476473442e5f5297459e8f1","avatarUrl":"/avatars/3fe8549dfd05a167d5b7b0287d5a7d46.svg","isPro":false,"fullname":"Gege Gao","user":"GGGHSL","type":"user","name":"GGGHSL"},"name":"Gege Gao","status":"admin_assigned","statusLastChangedAt":"2023-12-04T09:46:08.169Z","hidden":false},{"_id":"656d30f1f7be0986b4b5a352","user":{"_id":"648905d1a15c43c791d4381f","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/648905d1a15c43c791d4381f/GpqGBzsLiMHX0gWZEz3qn.jpeg","isPro":false,"fullname":"Weiyang Liu","user":"wy1iu","type":"user","name":"wy1iu"},"name":"Weiyang Liu","status":"admin_assigned","statusLastChangedAt":"2023-12-04T09:39:15.183Z","hidden":false},{"_id":"656d30f1f7be0986b4b5a353","user":{"_id":"63ea4b4be7f148f46015099f","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/63ea4b4be7f148f46015099f/tTuZYvtmM2IIGBH8BpKA1.jpeg","isPro":false,"fullname":"ANPEI CHEN","user":"apchen","type":"user","name":"apchen"},"name":"Anpei Chen","status":"admin_assigned","statusLastChangedAt":"2023-12-04T09:39:30.232Z","hidden":false},{"_id":"656d30f1f7be0986b4b5a354","user":{"_id":"620cae049086f3c07f01e3d5","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1644998139538-noauth.jpeg","isPro":false,"fullname":"Andreas Geiger","user":"andreas-geiger","type":"user","name":"andreas-geiger"},"name":"Andreas Geiger","status":"admin_assigned","statusLastChangedAt":"2023-12-04T09:39:37.165Z","hidden":false},{"_id":"656d30f1f7be0986b4b5a355","name":"Bernhard Schölkopf","hidden":false}],"publishedAt":"2023-11-30T18:59:58.000Z","title":"GraphDreamer: Compositional 3D Scene Synthesis from Scene Graphs","summary":"As pretrained text-to-image diffusion models become increasingly powerful,\nrecent efforts have been made to distill knowledge from these text-to-image\npretrained models for optimizing a text-guided 3D model. Most of the existing\nmethods generate a holistic 3D model from a plain text input. This can be\nproblematic when the text describes a complex scene with multiple objects,\nbecause the vectorized text embeddings are inherently unable to capture a\ncomplex description with multiple entities and relationships. Holistic 3D\nmodeling of the entire scene further prevents accurate grounding of text\nentities and concepts. To address this limitation, we propose GraphDreamer, a\nnovel framework to generate compositional 3D scenes from scene graphs, where\nobjects are represented as nodes and their interactions as edges. By exploiting\nnode and edge information in scene graphs, our method makes better use of the\npretrained text-to-image diffusion model and is able to fully disentangle\ndifferent objects without image-level supervision. To facilitate modeling of\nobject-wise relationships, we use signed distance fields as representation and\nimpose a constraint to avoid inter-penetration of objects. To avoid manual\nscene graph creation, we design a text prompt for ChatGPT to generate scene\ngraphs based on text inputs. We conduct both qualitative and quantitative\nexperiments to validate the effectiveness of GraphDreamer in generating\nhigh-fidelity compositional 3D scenes with disentangled object entities.","upvotes":16,"discussionId":"656d30f3f7be0986b4b5a3ca","ai_summary":"GraphDreamer generates complex 3D scenes from text inputs by using scene graphs to disentangle object entities and relationships, leveraging pretrained text-to-image diffusion models.","ai_keywords":["pretrained text-to-image diffusion models","scene graphs","nodes","edges","text-to-image diffusion model","signed distance fields","inter-penetration","ChatGPT"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2023-11-30T13:59:58.000Z","title":"GraphDreamer: Compositional 3D Scene Synthesis from Scene Graphs","summary":"As pretrained text-to-image diffusion models become increasingly powerful,\nrecent efforts have been made to distill knowledge from these text-to-image\npretrained models for optimizing a text-guided 3D model. Most of the existing\nmethods generate a holistic 3D model from a plain text input. This can be\nproblematic when the text describes a complex scene with multiple objects,\nbecause the vectorized text embeddings are inherently unable to capture a\ncomplex description with multiple entities and relationships. Holistic 3D\nmodeling of the entire scene further prevents accurate grounding of text\nentities and concepts. To address this limitation, we propose GraphDreamer, a\nnovel framework to generate compositional 3D scenes from scene graphs, where\nobjects are represented as nodes and their interactions as edges. By exploiting\nnode and edge information in scene graphs, our method makes better use of the\npretrained text-to-image diffusion model and is able to fully disentangle\ndifferent objects without image-level supervision. To facilitate modeling of\nobject-wise relationships, we use signed distance fields as representation and\nimpose a constraint to avoid inter-penetration of objects. To avoid manual\nscene graph creation, we design a text prompt for ChatGPT to generate scene\ngraphs based on text inputs. We conduct both qualitative and quantitative\nexperiments to validate the effectiveness of GraphDreamer in generating\nhigh-fidelity compositional 3D scenes with disentangled object entities.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2312.00093.png","numComments":1,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"GraphDreamer: Compositional 3D Scene Synthesis from Scene Graphs"}],"highlightedSummary":[{"type":"text","text":"As pretrained text-to-image diffusion models become increasingly powerful,\nrecent efforts have been made to distill knowledge from these text-to-image\npretrained models for optimizing a text-guided 3D model. Most of the existing\nmethods generate a holistic 3D model from a plain text input. This can be\nproblematic when the text describes a complex scene with multiple objects,\nbecause the vectorized text embeddings are inherently unable to capture a\ncomplex description with multiple entities and relationships. Holistic 3D\nmodeling of the entire scene further prevents accurate grounding of text\nentities and concepts. "},{"type":"highlight","text":"To address this limitation, we propose GraphDreamer, a\nnovel framework to generate compositional 3D scenes from scene graphs, where\nobjects are represented as nodes and their interactions as edges."},{"type":"text","text":" By exploiting\nnode and edge information in scene graphs, our method makes better use of the\npretrained text-to-image diffusion model and is able to fully disentangle\ndifferent objects without image-level supervision. To facilitate modeling of\nobject-wise relationships, we use signed distance fields as representation and\nimpose a constraint to avoid inter-penetration of objects. To avoid manual\nscene graph creation, we design a text prompt for ChatGPT to generate scene\ngraphs based on text inputs. We conduct both qualitative and quantitative\nexperiments to validate the effectiveness of GraphDreamer in generating\nhigh-fidelity compositional 3D scenes with disentangled object entities."}]},{"paper":{"id":"2412.12091","authors":[{"_id":"67610a192504d8a0fbf6a83a","user":{"_id":"64cb2227e984d09bed90fdc5","avatarUrl":"/avatars/15b04cfbd935bd01c84020c8a8e6c8c3.svg","isPro":false,"fullname":"Hanwen Liang","user":"hw-liang","type":"user","name":"hw-liang"},"name":"Hanwen Liang","status":"claimed_verified","statusLastChangedAt":"2024-12-18T09:53:00.045Z","hidden":false},{"_id":"67610a192504d8a0fbf6a83b","user":{"_id":"63f54aa73aa49d8cb97b84bc","avatarUrl":"/avatars/c73c5870039611ab9162daad46a1ba20.svg","isPro":false,"fullname":"junli cao","user":"jlcao2","type":"user","name":"jlcao2"},"name":"Junli Cao","status":"admin_assigned","statusLastChangedAt":"2024-12-17T11:19:05.096Z","hidden":false},{"_id":"67610a192504d8a0fbf6a83c","user":{"_id":"636c0c1a15cd58e915bb8139","avatarUrl":"/avatars/7c675ac6a7d303d3425e498c4e939eb0.svg","isPro":false,"fullname":"Vidit Goel","user":"vidit98","type":"user","name":"vidit98"},"name":"Vidit Goel","status":"admin_assigned","statusLastChangedAt":"2024-12-17T11:19:00.007Z","hidden":false},{"_id":"67610a192504d8a0fbf6a83d","user":{"_id":"645fed74335c21d19f3bf76c","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/645fed74335c21d19f3bf76c/gwVsllRWtSHbg4a1erkdF.jpeg","isPro":false,"fullname":"Guocheng Gordon Qian","user":"guochengqian","type":"user","name":"guochengqian"},"name":"Guocheng Qian","status":"admin_assigned","statusLastChangedAt":"2024-12-17T11:18:51.075Z","hidden":false},{"_id":"67610a192504d8a0fbf6a83e","user":{"_id":"6338047fe1b78345008dcd22","avatarUrl":"/avatars/7306138e83ca642f1638a39efa169c7f.svg","isPro":false,"fullname":"Sergei Korolev","user":"ZanyRumata","type":"user","name":"ZanyRumata"},"name":"Sergei Korolev","status":"admin_assigned","statusLastChangedAt":"2024-12-17T11:18:45.439Z","hidden":false},{"_id":"67610a192504d8a0fbf6a83f","name":"Demetri Terzopoulos","hidden":false},{"_id":"67610a192504d8a0fbf6a840","name":"Konstantinos N. Plataniotis","hidden":false},{"_id":"67610a192504d8a0fbf6a841","name":"Sergey Tulyakov","hidden":false},{"_id":"67610a192504d8a0fbf6a842","user":{"_id":"61f19829233c91cbd2f79e70","avatarUrl":"/avatars/a0735a94542b4f7cda5aed8bc4be0538.svg","isPro":false,"fullname":"Jian Ren","user":"alanspike","type":"user","name":"alanspike"},"name":"Jian Ren","status":"claimed_verified","statusLastChangedAt":"2025-01-15T08:49:35.742Z","hidden":false}],"publishedAt":"2024-12-16T18:58:17.000Z","title":"Wonderland: Navigating 3D Scenes from a Single Image","summary":"This paper addresses a challenging question: How can we efficiently create\nhigh-quality, wide-scope 3D scenes from a single arbitrary image? Existing\nmethods face several constraints, such as requiring multi-view data,\ntime-consuming per-scene optimization, low visual quality in backgrounds, and\ndistorted reconstructions in unseen areas. We propose a novel pipeline to\novercome these limitations. Specifically, we introduce a large-scale\nreconstruction model that uses latents from a video diffusion model to predict\n3D Gaussian Splattings for the scenes in a feed-forward manner. The video\ndiffusion model is designed to create videos precisely following specified\ncamera trajectories, allowing it to generate compressed video latents that\ncontain multi-view information while maintaining 3D consistency. We train the\n3D reconstruction model to operate on the video latent space with a progressive\ntraining strategy, enabling the efficient generation of high-quality,\nwide-scope, and generic 3D scenes. Extensive evaluations across various\ndatasets demonstrate that our model significantly outperforms existing methods\nfor single-view 3D scene generation, particularly with out-of-domain images.\nFor the first time, we demonstrate that a 3D reconstruction model can be\neffectively built upon the latent space of a diffusion model to realize\nefficient 3D scene generation.","upvotes":16,"discussionId":"67610a1a2504d8a0fbf6a87a","projectPage":"https://snap-research.github.io/wonderland/","ai_summary":"A novel pipeline using latents from a video diffusion model to predict 3D scenes from single images achieves high quality and efficiency, outperforming existing methods.","ai_keywords":["video diffusion model","3D Gaussian Splattings","feed-forward","3D reconstruction model","video latent space","progressive training","single-view 3D scene generation","out-of-domain images"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-12-16T13:58:17.000Z","title":"Wonderland: Navigating 3D Scenes from a Single Image","summary":"This paper addresses a challenging question: How can we efficiently create\nhigh-quality, wide-scope 3D scenes from a single arbitrary image? Existing\nmethods face several constraints, such as requiring multi-view data,\ntime-consuming per-scene optimization, low visual quality in backgrounds, and\ndistorted reconstructions in unseen areas. We propose a novel pipeline to\novercome these limitations. Specifically, we introduce a large-scale\nreconstruction model that uses latents from a video diffusion model to predict\n3D Gaussian Splattings for the scenes in a feed-forward manner. The video\ndiffusion model is designed to create videos precisely following specified\ncamera trajectories, allowing it to generate compressed video latents that\ncontain multi-view information while maintaining 3D consistency. We train the\n3D reconstruction model to operate on the video latent space with a progressive\ntraining strategy, enabling the efficient generation of high-quality,\nwide-scope, and generic 3D scenes. Extensive evaluations across various\ndatasets demonstrate that our model significantly outperforms existing methods\nfor single-view 3D scene generation, particularly with out-of-domain images.\nFor the first time, we demonstrate that a 3D reconstruction model can be\neffectively built upon the latent space of a diffusion model to realize\nefficient 3D scene generation.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2412.12091.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Wonderland: Navigating 3D Scenes from a Single Image"}],"highlightedSummary":[{"type":"text","text":"This paper addresses a challenging question: How can we efficiently create\nhigh-quality, wide-scope 3D scenes from a single arbitrary image? Existing\nmethods face several constraints, such as requiring multi-view data,\ntime-consuming per-scene optimization, low visual quality in backgrounds, and\ndistorted reconstructions in unseen areas. We propose a novel pipeline to\novercome these limitations. Specifically, we introduce a large-scale\nreconstruction model that uses latents from a video diffusion model to predict\n3D Gaussian Splattings for the scenes in a feed-forward manner. The video\ndiffusion model is designed to create videos precisely following specified\ncamera trajectories, allowing it to generate compressed video latents that\ncontain multi-view information while maintaining 3D consistency. We train the\n3D reconstruction model to operate on the video latent space with a progressive\ntraining strategy, enabling the efficient generation of high-quality,\nwide-scope, and generic 3D scenes. "},{"type":"highlight","text":"Extensive evaluations across various\ndatasets demonstrate that our model significantly outperforms existing methods\nfor single-view 3D scene generation, particularly with out-of-domain images."},{"type":"text","text":"\nFor the first time, we demonstrate that a 3D reconstruction model can be\neffectively built upon the latent space of a diffusion model to realize\nefficient 3D scene generation."}]},{"paper":{"id":"2401.05750","authors":[{"_id":"65d2a07a322cd4765ebab982","name":"Peng Dai","hidden":false},{"_id":"65d2a07a322cd4765ebab983","name":"Feitong Tan","hidden":false},{"_id":"65d2a07a322cd4765ebab984","user":{"_id":"62d1a71ac40bde8b986c3748","avatarUrl":"/avatars/6dbfba55563c71ed0db0d59c2629d525.svg","isPro":false,"fullname":"Xin Yu","user":"Andyx","type":"user","name":"Andyx"},"name":"Xin Yu","status":"claimed_verified","statusLastChangedAt":"2025-03-12T15:26:12.961Z","hidden":false},{"_id":"65d2a07a322cd4765ebab985","name":"Yinda Zhang","hidden":false},{"_id":"65d2a07a322cd4765ebab986","user":{"_id":"6875266f9cd3191dfddc7071","avatarUrl":"/avatars/64c581910833b111e9a7bae5b8740229.svg","isPro":false,"fullname":"xiaojuan qi","user":"xjqi","type":"user","name":"xjqi"},"name":"Xiaojuan Qi","status":"claimed_verified","statusLastChangedAt":"2025-07-15T19:15:58.850Z","hidden":false}],"publishedAt":"2024-01-11T08:58:13.000Z","title":"GO-NeRF: Generating Virtual Objects in Neural Radiance Fields","summary":"Despite advances in 3D generation, the direct creation of 3D objects within\nan existing 3D scene represented as NeRF remains underexplored. This process\nrequires not only high-quality 3D object generation but also seamless\ncomposition of the generated 3D content into the existing NeRF. To this end, we\npropose a new method, GO-NeRF, capable of utilizing scene context for\nhigh-quality and harmonious 3D object generation within an existing NeRF. Our\nmethod employs a compositional rendering formulation that allows the generated\n3D objects to be seamlessly composited into the scene utilizing learned\n3D-aware opacity maps without introducing unintended scene modification.\nMoreover, we also develop tailored optimization objectives and training\nstrategies to enhance the model's ability to exploit scene context and mitigate\nartifacts, such as floaters, originating from 3D object generation within a\nscene. Extensive experiments on both feed-forward and 360^o scenes show the\nsuperior performance of our proposed GO-NeRF in generating objects harmoniously\ncomposited with surrounding scenes and synthesizing high-quality novel view\nimages. Project page at {https://daipengwa.github.io/GO-NeRF/.","upvotes":0,"discussionId":"65d2a07d322cd4765ebaba0f","ai_summary":"GO-NeRF generates high-quality 3D objects within NeRF scenes using compositional rendering and learned opacity maps, achieving seamless integration and enhanced scene context.","ai_keywords":["NeRF","GO-NeRF","compositional rendering","3D-aware opacity maps","scene context","optimization objectives","training strategies","feed-forward scenes","$360^o$ scenes","novel view images"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-01-11T03:58:13.000Z","title":"GO-NeRF: Generating Virtual Objects in Neural Radiance Fields","summary":"Despite advances in 3D generation, the direct creation of 3D objects within\nan existing 3D scene represented as NeRF remains underexplored. This process\nrequires not only high-quality 3D object generation but also seamless\ncomposition of the generated 3D content into the existing NeRF. To this end, we\npropose a new method, GO-NeRF, capable of utilizing scene context for\nhigh-quality and harmonious 3D object generation within an existing NeRF. Our\nmethod employs a compositional rendering formulation that allows the generated\n3D objects to be seamlessly composited into the scene utilizing learned\n3D-aware opacity maps without introducing unintended scene modification.\nMoreover, we also develop tailored optimization objectives and training\nstrategies to enhance the model's ability to exploit scene context and mitigate\nartifacts, such as floaters, originating from 3D object generation within a\nscene. Extensive experiments on both feed-forward and 360^o scenes show the\nsuperior performance of our proposed GO-NeRF in generating objects harmoniously\ncomposited with surrounding scenes and synthesizing high-quality novel view\nimages. Project page at {https://daipengwa.github.io/GO-NeRF/.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2401.05750.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"GO-NeRF: Generating Virtual Objects in Neural Radiance Fields"}],"highlightedSummary":[{"type":"text","text":"Despite advances in 3D generation, the direct creation of 3D objects within\nan existing 3D scene represented as NeRF remains underexplored. This process\nrequires not only high-quality 3D object generation but also seamless\ncomposition of the generated 3D content into the existing NeRF. To this end, we\npropose a new method, GO-NeRF, capable of utilizing scene context for\nhigh-quality and harmonious 3D object generation within an existing NeRF. "},{"type":"highlight","text":"Our\nmethod employs a compositional rendering formulation that allows the generated\n3D objects to be seamlessly composited into the scene utilizing learned\n3D-aware opacity maps without introducing unintended scene modification."},{"type":"text","text":"\nMoreover, we also develop tailored optimization objectives and training\nstrategies to enhance the model's ability to exploit scene context and mitigate\nartifacts, such as floaters, originating from 3D object generation within a\nscene. Extensive experiments on both feed-forward and 360^o scenes show the\nsuperior performance of our proposed GO-NeRF in generating objects harmoniously\ncomposited with surrounding scenes and synthesizing high-quality novel view\nimages. Project page at {https://daipengwa.github.io/GO-NeRF/."}]},{"paper":{"id":"2607.11673","authors":[{"_id":"6a55f2a4a9d74d6e65bbd976","name":"Mingchao Sun","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd977","name":"Luyang Tang","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd978","name":"Yu Liu","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd979","name":"Xu Yan","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd97a","name":"Zhan Li","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd97b","name":"Yunwei Zhang","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd97c","name":"Fei Yu","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd97d","name":"Zengye Ge","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd97e","name":"Yumin Liu","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd97f","name":"Jiacheng Zhang","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd980","name":"Yongchang Zhang","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd981","name":"Jiawei Zhang","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd982","name":"Zhicheng Liu","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd983","name":"Zhongxu Sun","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd984","name":"Tianjian Ouyang","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd985","name":"Wenzheng Chen","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd986","name":"Shixing Yang","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd987","name":"Nianfei Fan","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd988","name":"Guodong Sun","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd989","name":"Huan Li","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd98a","name":"Zheng Zhou","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd98b","name":"Yongze Li","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd98c","name":"Yingliang Peng","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd98d","name":"Mengmeng Du","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd98e","name":"Yuan Liu","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd98f","name":"Haozhe Shi","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd990","name":"Chunnuo Gong","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd991","name":"Chengzhen Yu","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd992","name":"Chunxue Jia","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd993","name":"Yang Liu","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd994","name":"Shiying Zeng","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd995","name":"Junnan Lai","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd996","name":"Hang Zhang","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd997","name":"Ning Guo","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd998","name":"Baoquan Chen","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd999","name":"Mu Xu","hidden":false},{"_id":"6a55f2a4a9d74d6e65bbd99a","name":"Hongyu Pan","hidden":false}],"publishedAt":"2026-07-14T00:00:00.000Z","title":"ABot-3DWorld 0: A Universal World Model to Explore Any 3D Space","summary":"We present ABot-3DWorld 0, a universal multimodal 3D world model that turns text, image, and video inputs into high-fidelity, explorable 3D worlds. At the heart of our framework is a unified Spatial Generative Primitive (SGP), a compact tuple of a high-quality panorama and a spatial point cloud that delivers an efficient description of any 3D space. Multimodal inputs are first lifted into this primitive; a 3D-consistent panoramic video generator then explores the primitive along a planned trajectory; finally, our panoramic video reconstruction engine converts the generated video into a clean, photorealistic 3D Gaussian Splatting (3DGS) world. This pipeline covers two regimes: rich inputs (multi-view sets, casual video) are lifted into the SGP through a geometry-rigorous recovery that mirrors the observed scene, while a single image or sentence is completed generatively into a creative world. The result is one low-barrier engine for general 3D content creation that further anchors generated worlds to geographic points of interest, enabling map-native spatial exploration at consumer scale. Experiments show that ABot-3DWorld 0 sets the state of the art among open-source methods and demonstrates stronger scene fidelity than Marble under rich multimodal inputs.","upvotes":2,"discussionId":"6a55f2a4a9d74d6e65bbd99b","ai_summary":"ABot-3DWorld 0 converts text, images, and video into explorable 3D worlds via a spatial generative primitive and 3D Gaussian Splatting reconstruction.","ai_keywords":["Spatial Generative Primitive (SGP)","panoramic video generator","3D Gaussian Splatting (3DGS)","geometry-rigorous recovery","multimodal inputs","panoramic video reconstruction"],"ai_summary_model":"thinkingmachines/Inkling-Small"},"publishedAt":"2026-07-13T20:00:00.000Z","title":"ABot-3DWorld 0: A Universal World Model to Explore Any 3D Space","summary":"We present ABot-3DWorld 0, a universal multimodal 3D world model that turns text, image, and video inputs into high-fidelity, explorable 3D worlds. At the heart of our framework is a unified Spatial Generative Primitive (SGP), a compact tuple of a high-quality panorama and a spatial point cloud that delivers an efficient description of any 3D space. Multimodal inputs are first lifted into this primitive; a 3D-consistent panoramic video generator then explores the primitive along a planned trajectory; finally, our panoramic video reconstruction engine converts the generated video into a clean, photorealistic 3D Gaussian Splatting (3DGS) world. This pipeline covers two regimes: rich inputs (multi-view sets, casual video) are lifted into the SGP through a geometry-rigorous recovery that mirrors the observed scene, while a single image or sentence is completed generatively into a creative world. The result is one low-barrier engine for general 3D content creation that further anchors generated worlds to geographic points of interest, enabling map-native spatial exploration at consumer scale. Experiments show that ABot-3DWorld 0 sets the state of the art among open-source methods and demonstrates stronger scene fidelity than Marble under rich multimodal inputs.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2607.11673.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"ABot-3DWorld 0: A Universal World Model to Explore Any 3D Space"}],"highlightedSummary":[{"type":"text","text":"We present ABot-3DWorld 0, a universal multimodal 3D world model that turns text, image, and video inputs into high-fidelity, explorable 3D worlds. At the heart of our framework is a unified Spatial Generative Primitive (SGP), a compact tuple of a high-quality panorama and a spatial point cloud that delivers an efficient description of any 3D space. Multimodal inputs are first lifted into this primitive; a 3D-consistent panoramic video generator then explores the primitive along a planned trajectory; finally, our panoramic video reconstruction engine converts the generated video into a clean, photorealistic 3D Gaussian Splatting (3DGS) world. "},{"type":"highlight","text":"This pipeline covers two regimes: rich inputs (multi-view sets, casual video) are lifted into the SGP through a geometry-rigorous recovery that mirrors the observed scene, while a single image or sentence is completed generatively into a creative world."},{"type":"text","text":" The result is one low-barrier engine for general 3D content creation that further anchors generated worlds to geographic points of interest, enabling map-native spatial exploration at consumer scale. Experiments show that ABot-3DWorld 0 sets the state of the art among open-source methods and demonstrates stronger scene fidelity than Marble under rich multimodal inputs."}]},{"paper":{"id":"2408.15708","authors":[{"_id":"66cfdb63be74f57bd49d39bd","name":"Xinyu Gao","hidden":false},{"_id":"66cfdb63be74f57bd49d39be","user":{"_id":"65d6b7f9e77bb42e9a652c7f","avatarUrl":"/avatars/defa84e5df00a6599c217daffbcd4e08.svg","isPro":false,"fullname":"Zyeir","user":"Ingra14m","type":"user","name":"Ingra14m"},"name":"Ziyi Yang","status":"extracted_pending","statusLastChangedAt":"2024-08-29T02:22:32.743Z","hidden":false},{"_id":"66cfdb63be74f57bd49d39bf","user":{"_id":"634938bcaa45636285efda4c","avatarUrl":"/avatars/eb2f8d45be6b8396b446a0bbeaba7870.svg","isPro":false,"fullname":"Bingchen Gong","user":"wenri","type":"user","name":"wenri"},"name":"Bingchen Gong","status":"extracted_confirmed","statusLastChangedAt":"2026-01-31T18:21:35.917Z","hidden":false},{"_id":"66cfdb63be74f57bd49d39c0","name":"Xiaoguang Han","hidden":false},{"_id":"66cfdb63be74f57bd49d39c1","name":"Sipeng Yang","hidden":false},{"_id":"66cfdb63be74f57bd49d39c2","name":"Xiaogang Jin","hidden":false}],"publishedAt":"2024-08-28T11:13:27.000Z","title":"Towards Realistic Example-based Modeling via 3D Gaussian Stitching","summary":"Using parts of existing models to rebuild new models, commonly termed as\nexample-based modeling, is a classical methodology in the realm of computer\ngraphics. Previous works mostly focus on shape composition, making them very\nhard to use for realistic composition of 3D objects captured from real-world\nscenes. This leads to combining multiple NeRFs into a single 3D scene to\nachieve seamless appearance blending. However, the current SeamlessNeRF method\nstruggles to achieve interactive editing and harmonious stitching for\nreal-world scenes due to its gradient-based strategy and grid-based\nrepresentation. To this end, we present an example-based modeling method that\ncombines multiple Gaussian fields in a point-based representation using\nsample-guided synthesis. Specifically, as for composition, we create a GUI to\nsegment and transform multiple fields in real time, easily obtaining a\nsemantically meaningful composition of models represented by 3D Gaussian\nSplatting (3DGS). For texture blending, due to the discrete and irregular\nnature of 3DGS, straightforwardly applying gradient propagation as SeamlssNeRF\nis not supported. Thus, a novel sampling-based cloning method is proposed to\nharmonize the blending while preserving the original rich texture and content.\nOur workflow consists of three steps: 1) real-time segmentation and\ntransformation of a Gaussian model using a well-tailored GUI, 2) KNN analysis\nto identify boundary points in the intersecting area between the source and\ntarget models, and 3) two-phase optimization of the target model using\nsampling-based cloning and gradient constraints. Extensive experimental results\nvalidate that our approach significantly outperforms previous works in terms of\nrealistic synthesis, demonstrating its practicality. More demos are available\nat https://ingra14m.github.io/gs_stitching_website.","upvotes":8,"discussionId":"66cfdb68be74f57bd49d3bf9","ai_summary":"A novel example-based modeling method combines multiple Gaussian fields using a point-based representation for realistic 3D object composition and texture blending, outperforming previous approaches.","ai_keywords":["Gaussian fields","point-based representation","sample-guided synthesis","3D Gaussian Splatting (3DGS)","KNN analysis","sampling-based cloning","gradient constraints"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-08-28T07:13:27.000Z","title":"Towards Realistic Example-based Modeling via 3D Gaussian Stitching","summary":"Using parts of existing models to rebuild new models, commonly termed as\nexample-based modeling, is a classical methodology in the realm of computer\ngraphics. Previous works mostly focus on shape composition, making them very\nhard to use for realistic composition of 3D objects captured from real-world\nscenes. This leads to combining multiple NeRFs into a single 3D scene to\nachieve seamless appearance blending. However, the current SeamlessNeRF method\nstruggles to achieve interactive editing and harmonious stitching for\nreal-world scenes due to its gradient-based strategy and grid-based\nrepresentation. To this end, we present an example-based modeling method that\ncombines multiple Gaussian fields in a point-based representation using\nsample-guided synthesis. Specifically, as for composition, we create a GUI to\nsegment and transform multiple fields in real time, easily obtaining a\nsemantically meaningful composition of models represented by 3D Gaussian\nSplatting (3DGS). For texture blending, due to the discrete and irregular\nnature of 3DGS, straightforwardly applying gradient propagation as SeamlssNeRF\nis not supported. Thus, a novel sampling-based cloning method is proposed to\nharmonize the blending while preserving the original rich texture and content.\nOur workflow consists of three steps: 1) real-time segmentation and\ntransformation of a Gaussian model using a well-tailored GUI, 2) KNN analysis\nto identify boundary points in the intersecting area between the source and\ntarget models, and 3) two-phase optimization of the target model using\nsampling-based cloning and gradient constraints. Extensive experimental results\nvalidate that our approach significantly outperforms previous works in terms of\nrealistic synthesis, demonstrating its practicality. More demos are available\nat https://ingra14m.github.io/gs_stitching_website.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2408.15708.png","numComments":3,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Towards Realistic Example-based Modeling via 3D Gaussian Stitching"}],"highlightedSummary":[{"type":"text","text":"Using parts of existing models to rebuild new models, commonly termed as\nexample-based modeling, is a classical methodology in the realm of computer\ngraphics. "},{"type":"highlight","text":"Previous works mostly focus on shape composition, making them very\nhard to use for realistic composition of 3D objects captured from real-world\nscenes."},{"type":"text","text":" This leads to combining multiple NeRFs into a single 3D scene to\nachieve seamless appearance blending. However, the current SeamlessNeRF method\nstruggles to achieve interactive editing and harmonious stitching for\nreal-world scenes due to its gradient-based strategy and grid-based\nrepresentation. To this end, we present an example-based modeling method that\ncombines multiple Gaussian fields in a point-based representation using\nsample-guided synthesis. Specifically, as for composition, we create a GUI to\nsegment and transform multiple fields in real time, easily obtaining a\nsemantically meaningful composition of models represented by 3D Gaussian\nSplatting (3DGS). For texture blending, due to the discrete and irregular\nnature of 3DGS, straightforwardly applying gradient propagation as SeamlssNeRF\nis not supported. Thus, a novel sampling-based cloning method is proposed to\nharmonize the blending while preserving the original rich texture and content.\nOur workflow consists of three steps: 1) real-time segmentation and\ntransformation of a Gaussian model using a well-tailored GUI, 2) KNN analysis\nto identify boundary points in the intersecting area between the source and\ntarget models, and 3) two-phase optimization of the target model using\nsampling-based cloning and gradient constraints. Extensive experimental results\nvalidate that our approach significantly outperforms previous works in terms of\nrealistic synthesis, demonstrating its practicality. More demos are available\nat https://ingra14m.github.io/gs_stitching_website."}]},{"paper":{"id":"2603.29387","authors":[{"_id":"69cc9d3a3f84a0971be66bd7","user":{"_id":"66728afc4de1918e09192525","avatarUrl":"/avatars/9a2d444f0065cbdb5e40420e2a7cc64b.svg","isPro":true,"fullname":"Seungwoo Yoon","user":"SW-Yoon","type":"user","name":"SW-Yoon"},"name":"Seungwoo Yoon","status":"claimed_verified","statusLastChangedAt":"2026-04-01T08:34:26.484Z","hidden":false},{"_id":"69cc9d3a3f84a0971be66bd8","user":{"_id":"655b2c2edeee83130a571041","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/655b2c2edeee83130a571041/nOknykBsNLnKAW-VvogWL.jpeg","isPro":true,"fullname":"Jinmo Kim","user":"AImond","type":"user","name":"AImond"},"name":"Jinmo Kim","status":"claimed_verified","statusLastChangedAt":"2026-04-03T09:08:57.596Z","hidden":false},{"_id":"69cc9d3a3f84a0971be66bd9","name":"Jaesik Park","hidden":false}],"publishedAt":"2026-03-31T07:55:04.000Z","title":"Extend3D: Town-Scale 3D Generation","summary":"In this paper, we propose Extend3D, a training-free pipeline for 3D scene generation from a single image, built upon an object-centric 3D generative model. To overcome the limitations of fixed-size latent spaces in object-centric models for representing wide scenes, we extend the latent space in the x and y directions. Then, by dividing the extended latent space into overlapping patches, we apply the object-centric 3D generative model to each patch and couple them at each time step. Since patch-wise 3D generation with image conditioning requires strict spatial alignment between image and latent patches, we initialize the scene using a point cloud prior from a monocular depth estimator and iteratively refine occluded regions through SDEdit. We discovered that treating the incompleteness of 3D structure as noise during 3D refinement enables 3D completion via a concept, which we term under-noising. Furthermore, to address the sub-optimality of object-centric models for sub-scene generation, we optimize the extended latent during denoising, ensuring that the denoising trajectories remain consistent with the sub-scene dynamics. To this end, we introduce 3D-aware optimization objectives for improved geometric structure and texture fidelity. We demonstrate that our method yields better results than prior methods, as evidenced by human preference and quantitative experiments.","upvotes":23,"discussionId":"69cc9d3b3f84a0971be66bda","projectPage":"https://seungwoo-yoon.github.io/extend3d-page/","githubRepo":"https://github.com/SNU-VGILab/Extend3D","githubRepoAddedBy":"user","ai_summary":"An object-centric 3D generative model is extended with adaptive latent space and iterative refinement to generate complete 3D scenes from single images, incorporating noise-aware completion and 3D-aware optimization for improved fidelity.","ai_keywords":["object-centric 3D generative model","latent space extension","overlapping patches","SDEdit","under-noising","3D-aware optimization objectives","monocular depth estimator","denoising trajectories","geometric structure","texture fidelity"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":177,"organization":{"_id":"66d54dc8033492801db2bf5a","name":"SeoulNatlUniv","fullname":"Seoul National University","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/659ccc9d18897eb6594e897f/_-0BM-1UyM-d-lRiahFnf.png"}},"publishedAt":"2026-03-31T03:55:04.000Z","title":"Extend3D: Town-Scale 3D Generation","summary":"In this paper, we propose Extend3D, a training-free pipeline for 3D scene generation from a single image, built upon an object-centric 3D generative model. To overcome the limitations of fixed-size latent spaces in object-centric models for representing wide scenes, we extend the latent space in the x and y directions. Then, by dividing the extended latent space into overlapping patches, we apply the object-centric 3D generative model to each patch and couple them at each time step. Since patch-wise 3D generation with image conditioning requires strict spatial alignment between image and latent patches, we initialize the scene using a point cloud prior from a monocular depth estimator and iteratively refine occluded regions through SDEdit. We discovered that treating the incompleteness of 3D structure as noise during 3D refinement enables 3D completion via a concept, which we term under-noising. Furthermore, to address the sub-optimality of object-centric models for sub-scene generation, we optimize the extended latent during denoising, ensuring that the denoising trajectories remain consistent with the sub-scene dynamics. To this end, we introduce 3D-aware optimization objectives for improved geometric structure and texture fidelity. We demonstrate that our method yields better results than prior methods, as evidenced by human preference and quantitative experiments.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2603.29387.png","numComments":2,"upvoted":false,"organization":{"_id":"66d54dc8033492801db2bf5a","name":"SeoulNatlUniv","fullname":"Seoul National University","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/659ccc9d18897eb6594e897f/_-0BM-1UyM-d-lRiahFnf.png"},"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Extend3D: Town-Scale 3D Generation"}],"highlightedSummary":[{"type":"highlight","text":"In this paper, we propose Extend3D, a training-free pipeline for 3D scene generation from a single image, built upon an object-centric 3D generative model."},{"type":"text","text":" To overcome the limitations of fixed-size latent spaces in object-centric models for representing wide scenes, we extend the latent space in the x and y directions. Then, by dividing the extended latent space into overlapping patches, we apply the object-centric 3D generative model to each patch and couple them at each time step. Since patch-wise 3D generation with image conditioning requires strict spatial alignment between image and latent patches, we initialize the scene using a point cloud prior from a monocular depth estimator and iteratively refine occluded regions through SDEdit. We discovered that treating the incompleteness of 3D structure as noise during 3D refinement enables 3D completion via a concept, which we term under-noising. Furthermore, to address the sub-optimality of object-centric models for sub-scene generation, we optimize the extended latent during denoising, ensuring that the denoising trajectories remain consistent with the sub-scene dynamics. To this end, we introduce 3D-aware optimization objectives for improved geometric structure and texture fidelity. We demonstrate that our method yields better results than prior methods, as evidenced by human preference and quantitative experiments."}]},{"paper":{"id":"2501.06714","authors":[{"_id":"67d35fa77ca717ca8e503bca","user":{"_id":"6707f8800812a88fd6d87ebe","avatarUrl":"/avatars/28cd99359799ce85d5fe503144b776d8.svg","isPro":false,"fullname":"YUXIN WANG","user":"yuxinhk","type":"user","name":"yuxinhk"},"name":"Yuxin Wang","status":"claimed_verified","statusLastChangedAt":"2025-11-25T09:07:17.399Z","hidden":false},{"_id":"67d35fa77ca717ca8e503bcb","name":"Qianyi Wu","hidden":false},{"_id":"67d35fa77ca717ca8e503bcc","user":{"_id":"66feab48651e00e22f33222e","avatarUrl":"/avatars/7344377e2c796c7ec85194bb2fc78521.svg","isPro":false,"fullname":"Dan Xu","user":"danxuhk","type":"user","name":"danxuhk"},"name":"Dan Xu","status":"claimed_verified","statusLastChangedAt":"2025-03-17T14:42:38.630Z","hidden":false}],"publishedAt":"2025-01-12T04:44:44.000Z","title":"F3D-Gaus: Feed-forward 3D-aware Generation on ImageNet with\n  Cycle-Aggregative Gaussian Splatting","summary":"This paper tackles the problem of generalizable 3D-aware generation from\nmonocular datasets, e.g., ImageNet. The key challenge of this task is learning\na robust 3D-aware representation without multi-view or dynamic data, while\nensuring consistent texture and geometry across different viewpoints. Although\nsome baseline methods are capable of 3D-aware generation, the quality of the\ngenerated images still lags behind state-of-the-art 2D generation approaches,\nwhich excel in producing high-quality, detailed images. To address this severe\nlimitation, we propose a novel feed-forward pipeline based on pixel-aligned\nGaussian Splatting, coined as F3D-Gaus, which can produce more realistic and\nreliable 3D renderings from monocular inputs. In addition, we introduce a\nself-supervised cycle-aggregative constraint to enforce cross-view consistency\nin the learned 3D representation. This training strategy naturally allows\naggregation of multiple aligned Gaussian primitives and significantly\nalleviates the interpolation limitations inherent in single-view pixel-aligned\nGaussian Splatting. Furthermore, we incorporate video model priors to perform\ngeometry-aware refinement, enhancing the generation of fine details in\nwide-viewpoint scenarios and improving the model's capability to capture\nintricate 3D textures. Extensive experiments demonstrate that our approach not\nonly achieves high-quality, multi-view consistent 3D-aware generation from\nmonocular datasets, but also significantly improves training and inference\nefficiency.","upvotes":2,"discussionId":"67d35fab7ca717ca8e503d18","githubRepo":"https://github.com/W-Ted/F3D-Gaus","githubRepoAddedBy":"auto","ai_summary":"A novel feed-forward pipeline using pixel-aligned Gaussian Splatting and self-supervised cycle-aggregative constraints generates high-quality, multi-view consistent 3D renderings from monocular datasets.","ai_keywords":["feed-forward pipeline","pixel-aligned Gaussian Splatting","F3D-Gaus","self-supervised cycle-aggregative constraint","cross-view consistency","geometry-aware refinement","monocular datasets","3D-aware generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":52},"publishedAt":"2025-01-11T23:44:44.000Z","title":"F3D-Gaus: Feed-forward 3D-aware Generation on ImageNet with\n  Cycle-Aggregative Gaussian Splatting","summary":"This paper tackles the problem of generalizable 3D-aware generation from\nmonocular datasets, e.g., ImageNet. The key challenge of this task is learning\na robust 3D-aware representation without multi-view or dynamic data, while\nensuring consistent texture and geometry across different viewpoints. Although\nsome baseline methods are capable of 3D-aware generation, the quality of the\ngenerated images still lags behind state-of-the-art 2D generation approaches,\nwhich excel in producing high-quality, detailed images. To address this severe\nlimitation, we propose a novel feed-forward pipeline based on pixel-aligned\nGaussian Splatting, coined as F3D-Gaus, which can produce more realistic and\nreliable 3D renderings from monocular inputs. In addition, we introduce a\nself-supervised cycle-aggregative constraint to enforce cross-view consistency\nin the learned 3D representation. This training strategy naturally allows\naggregation of multiple aligned Gaussian primitives and significantly\nalleviates the interpolation limitations inherent in single-view pixel-aligned\nGaussian Splatting. Furthermore, we incorporate video model priors to perform\ngeometry-aware refinement, enhancing the generation of fine details in\nwide-viewpoint scenarios and improving the model's capability to capture\nintricate 3D textures. Extensive experiments demonstrate that our approach not\nonly achieves high-quality, multi-view consistent 3D-aware generation from\nmonocular datasets, but also significantly improves training and inference\nefficiency.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2501.06714.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"F3D-Gaus: Feed-forward 3D-aware Generation on ImageNet with\n  Cycle-Aggregative Gaussian Splatting"}],"highlightedSummary":[{"type":"highlight","text":"This paper tackles the problem of generalizable 3D-aware generation from\nmonocular datasets, e.g., ImageNet."},{"type":"text","text":" The key challenge of this task is learning\na robust 3D-aware representation without multi-view or dynamic data, while\nensuring consistent texture and geometry across different viewpoints. Although\nsome baseline methods are capable of 3D-aware generation, the quality of the\ngenerated images still lags behind state-of-the-art 2D generation approaches,\nwhich excel in producing high-quality, detailed images. To address this severe\nlimitation, we propose a novel feed-forward pipeline based on pixel-aligned\nGaussian Splatting, coined as F3D-Gaus, which can produce more realistic and\nreliable 3D renderings from monocular inputs. In addition, we introduce a\nself-supervised cycle-aggregative constraint to enforce cross-view consistency\nin the learned 3D representation. This training strategy naturally allows\naggregation of multiple aligned Gaussian primitives and significantly\nalleviates the interpolation limitations inherent in single-view pixel-aligned\nGaussian Splatting. Furthermore, we incorporate video model priors to perform\ngeometry-aware refinement, enhancing the generation of fine details in\nwide-viewpoint scenarios and improving the model's capability to capture\nintricate 3D textures. Extensive experiments demonstrate that our approach not\nonly achieves high-quality, multi-view consistent 3D-aware generation from\nmonocular datasets, but also significantly improves training and inference\nefficiency."}]},{"paper":{"id":"2506.05573","authors":[{"_id":"6846902a3ec10bdd8ab4db61","user":{"_id":"67b2b698ba726eda5cbd07da","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/noauth/qZvmHqsxV2lXfYz2Wx3rA.jpeg","isPro":false,"fullname":"Yuchen Lin","user":"wgsxm","type":"user","name":"wgsxm"},"name":"Yuchen Lin","status":"claimed_verified","statusLastChangedAt":"2025-06-10T09:21:53.647Z","hidden":false},{"_id":"6846902a3ec10bdd8ab4db62","user":{"_id":"62e18206926f4892a4c782bd","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/62e18206926f4892a4c782bd/m_OAeRb2a7a5PlnK6d4Dg.jpeg","isPro":false,"fullname":"Chenguo Lin","user":"chenguolin","type":"user","name":"chenguolin"},"name":"Chenguo Lin","status":"claimed_verified","statusLastChangedAt":"2025-06-09T10:11:09.640Z","hidden":false},{"_id":"6846902a3ec10bdd8ab4db63","user":{"_id":"654866e8cd0a5621395f8287","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/654866e8cd0a5621395f8287/4Bccwd1ehn-Ee4T1rId5S.jpeg","isPro":false,"fullname":"Panwang Pan","user":"paulpanwang","type":"user","name":"paulpanwang"},"name":"Panwang Pan","status":"claimed_verified","statusLastChangedAt":"2025-06-10T09:21:55.707Z","hidden":false},{"_id":"6846902a3ec10bdd8ab4db64","name":"Honglei Yan","hidden":false},{"_id":"6846902a3ec10bdd8ab4db65","name":"Yiqiang Feng","hidden":false},{"_id":"6846902a3ec10bdd8ab4db66","name":"Yadong Mu","hidden":false},{"_id":"6846902a3ec10bdd8ab4db67","name":"Katerina Fragkiadaki","hidden":false}],"publishedAt":"2025-06-05T20:30:28.000Z","title":"PartCrafter: Structured 3D Mesh Generation via Compositional Latent\n  Diffusion Transformers","summary":"We introduce PartCrafter, the first structured 3D generative model that\njointly synthesizes multiple semantically meaningful and geometrically distinct\n3D meshes from a single RGB image. Unlike existing methods that either produce\nmonolithic 3D shapes or follow two-stage pipelines, i.e., first segmenting an\nimage and then reconstructing each segment, PartCrafter adopts a unified,\ncompositional generation architecture that does not rely on pre-segmented\ninputs. Conditioned on a single image, it simultaneously denoises multiple 3D\nparts, enabling end-to-end part-aware generation of both individual objects and\ncomplex multi-object scenes. PartCrafter builds upon a pretrained 3D mesh\ndiffusion transformer (DiT) trained on whole objects, inheriting the pretrained\nweights, encoder, and decoder, and introduces two key innovations: (1) A\ncompositional latent space, where each 3D part is represented by a set of\ndisentangled latent tokens; (2) A hierarchical attention mechanism that enables\nstructured information flow both within individual parts and across all parts,\nensuring global coherence while preserving part-level detail during generation.\nTo support part-level supervision, we curate a new dataset by mining part-level\nannotations from large-scale 3D object datasets. Experiments show that\nPartCrafter outperforms existing approaches in generating decomposable 3D\nmeshes, including parts that are not directly visible in input images,\ndemonstrating the strength of part-aware generative priors for 3D understanding\nand synthesis. Code and training data will be released.","upvotes":83,"discussionId":"6846902a3ec10bdd8ab4db68","projectPage":"https://wgsxm.github.io/projects/partcrafter","githubRepo":"https://github.com/wgsxm/PartCrafter","githubRepoAddedBy":"user","ai_summary":"PartCrafter is a unified 3D generative model that synthesizes multiple semantically meaningful 3D meshes from a single image using a compositional latent space and hierarchical attention mechanism.","ai_keywords":["3D generative model","multiple 3D meshes","RGB image","unified compositional generation architecture","denoising","3D diffusion transformer (DiT)","compositional latent space","disentangled latent tokens","hierarchical attention mechanism","part-level supervision","part-aware generative priors"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":2485},"publishedAt":"2025-06-05T16:30:28.000Z","title":"PartCrafter: Structured 3D Mesh Generation via Compositional Latent\n  Diffusion Transformers","summary":"We introduce PartCrafter, the first structured 3D generative model that\njointly synthesizes multiple semantically meaningful and geometrically distinct\n3D meshes from a single RGB image. Unlike existing methods that either produce\nmonolithic 3D shapes or follow two-stage pipelines, i.e., first segmenting an\nimage and then reconstructing each segment, PartCrafter adopts a unified,\ncompositional generation architecture that does not rely on pre-segmented\ninputs. Conditioned on a single image, it simultaneously denoises multiple 3D\nparts, enabling end-to-end part-aware generation of both individual objects and\ncomplex multi-object scenes. PartCrafter builds upon a pretrained 3D mesh\ndiffusion transformer (DiT) trained on whole objects, inheriting the pretrained\nweights, encoder, and decoder, and introduces two key innovations: (1) A\ncompositional latent space, where each 3D part is represented by a set of\ndisentangled latent tokens; (2) A hierarchical attention mechanism that enables\nstructured information flow both within individual parts and across all parts,\nensuring global coherence while preserving part-level detail during generation.\nTo support part-level supervision, we curate a new dataset by mining part-level\nannotations from large-scale 3D object datasets. Experiments show that\nPartCrafter outperforms existing approaches in generating decomposable 3D\nmeshes, including parts that are not directly visible in input images,\ndemonstrating the strength of part-aware generative priors for 3D understanding\nand synthesis. Code and training data will be released.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2506.05573.png","numComments":8,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"PartCrafter: Structured 3D Mesh Generation via Compositional Latent\n  Diffusion Transformers"}],"highlightedSummary":[{"type":"text","text":"We introduce PartCrafter, the first structured 3D generative model that\njointly synthesizes multiple semantically meaningful and geometrically distinct\n3D meshes from a single RGB image. Unlike existing methods that either produce\nmonolithic 3D shapes or follow two-stage pipelines, i.e., first segmenting an\nimage and then reconstructing each segment, PartCrafter adopts a unified,\ncompositional generation architecture that does not rely on pre-segmented\ninputs. Conditioned on a single image, it simultaneously denoises multiple 3D\nparts, enabling end-to-end part-aware generation of both individual objects and\ncomplex multi-object scenes. PartCrafter builds upon a pretrained 3D mesh\ndiffusion transformer (DiT) trained on whole objects, inheriting the pretrained\nweights, encoder, and decoder, and introduces two key innovations: (1) A\ncompositional latent space, where each 3D part is represented by a set of\ndisentangled latent tokens; (2) A hierarchical attention mechanism that enables\nstructured information flow both within individual parts and across all parts,\nensuring global coherence while preserving part-level detail during generation.\nTo support part-level supervision, we curate a new dataset by mining part-level\nannotations from large-scale 3D object datasets. Experiments show that\nPartCrafter outperforms existing approaches in generating decomposable 3D\nmeshes, including parts that are not directly visible in input images,\ndemonstrating the strength of part-aware generative priors for 3D understanding\nand synthesis. Code and training data will be released."}]},{"paper":{"id":"2302.07224","authors":[{"_id":"6505cea97de4710bafcebadc","name":"Shangzhan Zhang","hidden":false},{"_id":"6505cea97de4710bafcebadd","name":"Sida Peng","hidden":false},{"_id":"6505cea97de4710bafcebade","name":"Tianrun Chen","hidden":false},{"_id":"6505cea97de4710bafcebadf","name":"Linzhan Mou","hidden":false},{"_id":"6505cea97de4710bafcebae0","name":"Haotong Lin","hidden":false},{"_id":"6505cea97de4710bafcebae1","name":"Kaicheng Yu","hidden":false},{"_id":"6505cea97de4710bafcebae2","name":"Yiyi Liao","hidden":false},{"_id":"6505cea97de4710bafcebae3","name":"Xiaowei Zhou","hidden":false}],"publishedAt":"2023-02-14T17:57:58.000Z","title":"Painting 3D Nature in 2D: View Synthesis of Natural Scenes from a Single\n  Semantic Mask","summary":"We introduce a novel approach that takes a single semantic mask as input to\nsynthesize multi-view consistent color images of natural scenes, trained with a\ncollection of single images from the Internet. Prior works on 3D-aware image\nsynthesis either require multi-view supervision or learning category-level\nprior for specific classes of objects, which can hardly work for natural\nscenes. Our key idea to solve this challenging problem is to use a semantic\nfield as the intermediate representation, which is easier to reconstruct from\nan input semantic mask and then translate to a radiance field with the\nassistance of off-the-shelf semantic image synthesis models. Experiments show\nthat our method outperforms baseline methods and produces photorealistic,\nmulti-view consistent videos of a variety of natural scenes.","upvotes":1,"discussionId":"6505ceb27de4710bafcebdcc","ai_summary":"A method using semantic fields and preexisting semantic synthesis models generates multi-view consistent natural scene videos from single semantic masks.","ai_keywords":["semantic mask","multi-view consistent","color images","3D-aware image synthesis","semantic field","radiance field","photorealistic"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2023-02-14T12:57:58.000Z","title":"Painting 3D Nature in 2D: View Synthesis of Natural Scenes from a Single\n  Semantic Mask","summary":"We introduce a novel approach that takes a single semantic mask as input to\nsynthesize multi-view consistent color images of natural scenes, trained with a\ncollection of single images from the Internet. Prior works on 3D-aware image\nsynthesis either require multi-view supervision or learning category-level\nprior for specific classes of objects, which can hardly work for natural\nscenes. Our key idea to solve this challenging problem is to use a semantic\nfield as the intermediate representation, which is easier to reconstruct from\nan input semantic mask and then translate to a radiance field with the\nassistance of off-the-shelf semantic image synthesis models. Experiments show\nthat our method outperforms baseline methods and produces photorealistic,\nmulti-view consistent videos of a variety of natural scenes.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2302.07224.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Painting 3D Nature in 2D: View Synthesis of Natural Scenes from a Single\n  Semantic Mask"}],"highlightedSummary":[{"type":"text","text":"We introduce a novel approach that takes a single semantic mask as input to\nsynthesize multi-view consistent color images of natural scenes, trained with a\ncollection of single images from the Internet. "},{"type":"highlight","text":"Prior works on 3D-aware image\nsynthesis either require multi-view supervision or learning category-level\nprior for specific classes of objects, which can hardly work for natural\nscenes."},{"type":"text","text":" Our key idea to solve this challenging problem is to use a semantic\nfield as the intermediate representation, which is easier to reconstruct from\nan input semantic mask and then translate to a radiance field with the\nassistance of off-the-shelf semantic image synthesis models. Experiments show\nthat our method outperforms baseline methods and produces photorealistic,\nmulti-view consistent videos of a variety of natural scenes."}]},{"paper":{"id":"2312.12419","authors":[{"_id":"6641aeea6b649b6aa34f39fd","name":"Jinghao Zhou","hidden":false},{"_id":"6641aeea6b649b6aa34f39fe","name":"Tomas Jakab","hidden":false},{"_id":"6641aeea6b649b6aa34f39ff","name":"Philip Torr","hidden":false},{"_id":"6641aeea6b649b6aa34f3a00","name":"Christian Rupprecht","hidden":false}],"publishedAt":"2023-12-19T18:50:33.000Z","title":"Scene-Conditional 3D Object Stylization and Composition","summary":"Recently, 3D generative models have made impressive progress, enabling the\ngeneration of almost arbitrary 3D assets from text or image inputs. However,\nthese approaches generate objects in isolation without any consideration for\nthe scene where they will eventually be placed. In this paper, we propose a\nframework that allows for the stylization of an existing 3D asset to fit into a\ngiven 2D scene, and additionally produce a photorealistic composition as if the\nasset was placed within the environment. This not only opens up a new level of\ncontrol for object stylization, for example, the same assets can be stylized to\nreflect changes in the environment, such as summer to winter or fantasy versus\nfuturistic settings-but also makes the object-scene composition more\ncontrollable. We achieve this by combining modeling and optimizing the object's\ntexture and environmental lighting through differentiable ray tracing with\nimage priors from pre-trained text-to-image diffusion models. We demonstrate\nthat our method is applicable to a wide variety of indoor and outdoor scenes\nand arbitrary objects.","upvotes":0,"discussionId":"6641aeef6b649b6aa34f3b32","ai_summary":"A framework combines differentiable ray tracing with pre-trained text-to-image diffusion models to stylize 3D assets for specific scenes and produce photorealistic compositions.","ai_keywords":["3D generative models","text-to-image diffusion models","differentiable ray tracing","image priors"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2023-12-19T13:50:33.000Z","title":"Scene-Conditional 3D Object Stylization and Composition","summary":"Recently, 3D generative models have made impressive progress, enabling the\ngeneration of almost arbitrary 3D assets from text or image inputs. However,\nthese approaches generate objects in isolation without any consideration for\nthe scene where they will eventually be placed. In this paper, we propose a\nframework that allows for the stylization of an existing 3D asset to fit into a\ngiven 2D scene, and additionally produce a photorealistic composition as if the\nasset was placed within the environment. This not only opens up a new level of\ncontrol for object stylization, for example, the same assets can be stylized to\nreflect changes in the environment, such as summer to winter or fantasy versus\nfuturistic settings-but also makes the object-scene composition more\ncontrollable. We achieve this by combining modeling and optimizing the object's\ntexture and environmental lighting through differentiable ray tracing with\nimage priors from pre-trained text-to-image diffusion models. We demonstrate\nthat our method is applicable to a wide variety of indoor and outdoor scenes\nand arbitrary objects.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2312.12419.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Scene-Conditional 3D Object Stylization and Composition"}],"highlightedSummary":[{"type":"text","text":"Recently, 3D generative models have made impressive progress, enabling the\ngeneration of almost arbitrary 3D assets from text or image inputs. However,\nthese approaches generate objects in isolation without any consideration for\nthe scene where they will eventually be placed. In this paper, we propose a\nframework that allows for the stylization of an existing 3D asset to fit into a\ngiven 2D scene, and additionally produce a photorealistic composition as if the\nasset was placed within the environment. This not only opens up a new level of\ncontrol for object stylization, for example, the same assets can be stylized to\nreflect changes in the environment, such as summer to winter or fantasy versus\nfuturistic settings-but also makes the object-scene composition more\ncontrollable. We achieve this by combining modeling and optimizing the object's\ntexture and environmental lighting through differentiable ray tracing with\nimage priors from pre-trained text-to-image diffusion models. "},{"type":"highlight","text":"We demonstrate\nthat our method is applicable to a wide variety of indoor and outdoor scenes\nand arbitrary objects."}]},{"paper":{"id":"2512.07348","authors":[{"_id":"693987753caaa02184ac09c7","name":"Xinyu Wei","hidden":false},{"_id":"693987753caaa02184ac09c8","name":"Kangrui Cen","hidden":false},{"_id":"693987753caaa02184ac09c9","name":"Hongyang Wei","hidden":false},{"_id":"693987753caaa02184ac09ca","name":"Zhen Guo","hidden":false},{"_id":"693987753caaa02184ac09cb","name":"Bairui Li","hidden":false},{"_id":"693987753caaa02184ac09cc","name":"Zeqing Wang","hidden":false},{"_id":"693987753caaa02184ac09cd","name":"Jinrui Zhang","hidden":false},{"_id":"693987753caaa02184ac09ce","name":"Lei Zhang","hidden":false}],"publishedAt":"2025-12-08T09:40:11.000Z","title":"MICo-150K: A Comprehensive Dataset Advancing Multi-Image Composition","summary":"In controllable image generation, synthesizing coherent and consistent images from multiple reference inputs, i.e., Multi-Image Composition (MICo), remains a challenging problem, partly hindered by the lack of high-quality training data. To bridge this gap, we conduct a systematic study of MICo, categorizing it into 7 representative tasks and curate a large-scale collection of high-quality source images and construct diverse MICo prompts. Leveraging powerful proprietary models, we synthesize a rich amount of balanced composite images, followed by human-in-the-loop filtering and refinement, resulting in MICo-150K, a comprehensive dataset for MICo with identity consistency. We further build a Decomposition-and-Recomposition (De&Re) subset, where 11K real-world complex images are decomposed into components and recomposed, enabling both real and synthetic compositions. To enable comprehensive evaluation, we construct MICo-Bench with 100 cases per task and 300 challenging De&Re cases, and further introduce a new metric, Weighted-Ref-VIEScore, specifically tailored for MICo evaluation. Finally, we fine-tune multiple models on MICo-150K and evaluate them on MICo-Bench. The results show that MICo-150K effectively equips models without MICo capability and further enhances those with existing skills. Notably, our baseline model, Qwen-MICo, fine-tuned from Qwen-Image-Edit, matches Qwen-Image-2509 in 3-image composition while supporting arbitrary multi-image inputs beyond the latter's limitation. Our dataset, benchmark, and baseline collectively offer valuable resources for further research on Multi-Image Composition.","upvotes":2,"discussionId":"693987753caaa02184ac09cf","ai_summary":"MICo-150K, a large-scale dataset with identity consistency, and MICo-Bench, a benchmark with a new metric, enable effective fine-tuning and evaluation for multi-image composition tasks.","ai_keywords":["Multi-Image Composition (MICo)","Decomposition-and-Recomposition (De&Re)","MICo-150K","MICo-Bench","Weighted-Ref-VIEScore","fine-tuning","Qwen-MICo","Qwen-Image-Edit","Qwen-Image-2509"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-12-08T04:40:11.000Z","title":"MICo-150K: A Comprehensive Dataset Advancing Multi-Image Composition","summary":"In controllable image generation, synthesizing coherent and consistent images from multiple reference inputs, i.e., Multi-Image Composition (MICo), remains a challenging problem, partly hindered by the lack of high-quality training data. To bridge this gap, we conduct a systematic study of MICo, categorizing it into 7 representative tasks and curate a large-scale collection of high-quality source images and construct diverse MICo prompts. Leveraging powerful proprietary models, we synthesize a rich amount of balanced composite images, followed by human-in-the-loop filtering and refinement, resulting in MICo-150K, a comprehensive dataset for MICo with identity consistency. We further build a Decomposition-and-Recomposition (De&Re) subset, where 11K real-world complex images are decomposed into components and recomposed, enabling both real and synthetic compositions. To enable comprehensive evaluation, we construct MICo-Bench with 100 cases per task and 300 challenging De&Re cases, and further introduce a new metric, Weighted-Ref-VIEScore, specifically tailored for MICo evaluation. Finally, we fine-tune multiple models on MICo-150K and evaluate them on MICo-Bench. The results show that MICo-150K effectively equips models without MICo capability and further enhances those with existing skills. Notably, our baseline model, Qwen-MICo, fine-tuned from Qwen-Image-Edit, matches Qwen-Image-2509 in 3-image composition while supporting arbitrary multi-image inputs beyond the latter's limitation. Our dataset, benchmark, and baseline collectively offer valuable resources for further research on Multi-Image Composition.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2512.07348.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"MICo-150K: A Comprehensive Dataset Advancing Multi-Image Composition"}],"highlightedSummary":[{"type":"highlight","text":"In controllable image generation, synthesizing coherent and consistent images from multiple reference inputs, i.e., Multi-Image Composition (MICo), remains a challenging problem, partly hindered by the lack of high-quality training data."},{"type":"text","text":" To bridge this gap, we conduct a systematic study of MICo, categorizing it into 7 representative tasks and curate a large-scale collection of high-quality source images and construct diverse MICo prompts. Leveraging powerful proprietary models, we synthesize a rich amount of balanced composite images, followed by human-in-the-loop filtering and refinement, resulting in MICo-150K, a comprehensive dataset for MICo with identity consistency. We further build a Decomposition-and-Recomposition (De&Re) subset, where 11K real-world complex images are decomposed into components and recomposed, enabling both real and synthetic compositions. To enable comprehensive evaluation, we construct MICo-Bench with 100 cases per task and 300 challenging De&Re cases, and further introduce a new metric, Weighted-Ref-VIEScore, specifically tailored for MICo evaluation. Finally, we fine-tune multiple models on MICo-150K and evaluate them on MICo-Bench. The results show that MICo-150K effectively equips models without MICo capability and further enhances those with existing skills. Notably, our baseline model, Qwen-MICo, fine-tuned from Qwen-Image-Edit, matches Qwen-Image-2509 in 3-image composition while supporting arbitrary multi-image inputs beyond the latter's limitation. Our dataset, benchmark, and baseline collectively offer valuable resources for further research on Multi-Image Composition."}]},{"paper":{"id":"2402.12908","authors":[{"_id":"65d56827543d28ba7d595852","user":{"_id":"653e5d31ffd60206c8b64bb5","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/653e5d31ffd60206c8b64bb5/bgztraPC27L6culMlJw4s.png","isPro":false,"fullname":"Xinchen Zhang","user":"comin","type":"user","name":"comin"},"name":"Xinchen Zhang","status":"claimed_verified","statusLastChangedAt":"2024-07-29T07:10:13.151Z","hidden":false},{"_id":"65d56827543d28ba7d595853","user":{"_id":"64fde4e252e82dd432b74ce9","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64fde4e252e82dd432b74ce9/-CQZbBP7FsPPyawYrsi4z.jpeg","isPro":false,"fullname":"Ling Yang","user":"Lingaaaaaaa","type":"user","name":"Lingaaaaaaa"},"name":"Ling Yang","status":"claimed_verified","statusLastChangedAt":"2024-07-12T07:42:08.820Z","hidden":false},{"_id":"65d56827543d28ba7d595854","user":{"_id":"657fe0b1f010d76b6e709110","avatarUrl":"/avatars/9c08817484cfb5b238d6833d788fa0a3.svg","isPro":false,"fullname":"YaQi Cai","user":"yaqicc","type":"user","name":"yaqicc"},"name":"Yaqi Cai","status":"admin_assigned","statusLastChangedAt":"2024-02-21T12:22:09.041Z","hidden":false},{"_id":"65d56827543d28ba7d595855","name":"Zhaochen Yu","hidden":false},{"_id":"65d56827543d28ba7d595856","name":"Jiake Xie","hidden":false},{"_id":"65d56827543d28ba7d595857","user":{"_id":"64e357dd825f4133e7427bf8","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64e357dd825f4133e7427bf8/HwaWhINrzkbXG6SHG2oyf.jpeg","isPro":false,"fullname":"tyfeld","user":"tyfeld","type":"user","name":"tyfeld"},"name":"Ye Tian","status":"claimed_verified","statusLastChangedAt":"2025-03-25T08:23:52.054Z","hidden":false},{"_id":"65d56827543d28ba7d595858","user":{"_id":"64c0e950aa57599de1c75dad","avatarUrl":"/avatars/374d53317cbccc30fae70e5152ca13e0.svg","isPro":false,"fullname":"Minkai Xu","user":"mkxu","type":"user","name":"mkxu"},"name":"Minkai Xu","status":"claimed_verified","statusLastChangedAt":"2024-03-05T20:57:45.302Z","hidden":false},{"_id":"65d56827543d28ba7d595859","name":"Yong Tang","hidden":false},{"_id":"65d56827543d28ba7d59585a","user":{"_id":"64ca1fe838837b12d5e529b7","avatarUrl":"/avatars/44a3ad9e59318784ac531993b5f69f6b.svg","isPro":false,"fullname":"Yujiu Yang","user":"Thu-redrobot","type":"user","name":"Thu-redrobot"},"name":"Yujiu Yang","status":"admin_assigned","statusLastChangedAt":"2024-02-21T12:20:56.658Z","hidden":false},{"_id":"65d56827543d28ba7d59585b","name":"Bin Cui","hidden":false}],"publishedAt":"2024-02-20T10:56:52.000Z","title":"RealCompo: Dynamic Equilibrium between Realism and Compositionality\n  Improves Text-to-Image Diffusion Models","summary":"Diffusion models have achieved remarkable advancements in text-to-image\ngeneration. However, existing models still have many difficulties when faced\nwith multiple-object compositional generation. In this paper, we propose a new\ntraining-free and transferred-friendly text-to-image generation framework,\nnamely RealCompo, which aims to leverage the advantages of text-to-image and\nlayout-to-image models to enhance both realism and compositionality of the\ngenerated images. An intuitive and novel balancer is proposed to dynamically\nbalance the strengths of the two models in denoising process, allowing\nplug-and-play use of any model without extra training. Extensive experiments\nshow that our RealCompo consistently outperforms state-of-the-art text-to-image\nmodels and layout-to-image models in multiple-object compositional generation\nwhile keeping satisfactory realism and compositionality of the generated\nimages. Code is available at https://github.com/YangLing0818/RealCompo","upvotes":9,"discussionId":"65d56828543d28ba7d5958b2","projectPage":"https://cominclip.github.io/RealCompo_Page/","githubRepo":"https://github.com/YangLing0818/RealCompo","githubRepoAddedBy":"user","ai_summary":"RealCompo is a training-free text-to-image generation framework that enhances realism and compositionality by dynamically balancing text-to-image and layout-to-image models.","ai_keywords":["diffusion models","text-to-image generation","layout-to-image models","multiple-object compositional generation","balancer","denoising process"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":122},"publishedAt":"2024-02-20T05:56:52.000Z","title":"RealCompo: Dynamic Equilibrium between Realism and Compositionality\n  Improves Text-to-Image Diffusion Models","summary":"Diffusion models have achieved remarkable advancements in text-to-image\ngeneration. However, existing models still have many difficulties when faced\nwith multiple-object compositional generation. In this paper, we propose a new\ntraining-free and transferred-friendly text-to-image generation framework,\nnamely RealCompo, which aims to leverage the advantages of text-to-image and\nlayout-to-image models to enhance both realism and compositionality of the\ngenerated images. An intuitive and novel balancer is proposed to dynamically\nbalance the strengths of the two models in denoising process, allowing\nplug-and-play use of any model without extra training. Extensive experiments\nshow that our RealCompo consistently outperforms state-of-the-art text-to-image\nmodels and layout-to-image models in multiple-object compositional generation\nwhile keeping satisfactory realism and compositionality of the generated\nimages. Code is available at https://github.com/YangLing0818/RealCompo","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2402.12908.png","numComments":1,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"RealCompo: Dynamic Equilibrium between Realism and Compositionality\n  Improves Text-to-Image Diffusion Models"}],"highlightedSummary":[{"type":"text","text":"Diffusion models have achieved remarkable advancements in text-to-image\ngeneration. "},{"type":"highlight","text":"However, existing models still have many difficulties when faced\nwith multiple-object compositional generation."},{"type":"text","text":" In this paper, we propose a new\ntraining-free and transferred-friendly text-to-image generation framework,\nnamely RealCompo, which aims to leverage the advantages of text-to-image and\nlayout-to-image models to enhance both realism and compositionality of the\ngenerated images. An intuitive and novel balancer is proposed to dynamically\nbalance the strengths of the two models in denoising process, allowing\nplug-and-play use of any model without extra training. Extensive experiments\nshow that our RealCompo consistently outperforms state-of-the-art text-to-image\nmodels and layout-to-image models in multiple-object compositional generation\nwhile keeping satisfactory realism and compositionality of the generated\nimages. Code is available at https://github.com/YangLing0818/RealCompo"}]},{"paper":{"id":"2506.00742","authors":[{"_id":"6847f1f388cb0eba3d0a27cc","name":"Zeqi Gu","hidden":false},{"_id":"6847f1f388cb0eba3d0a27cd","name":"Yin Cui","hidden":false},{"_id":"6847f1f388cb0eba3d0a27ce","name":"Zhaoshuo Li","hidden":false},{"_id":"6847f1f388cb0eba3d0a27cf","name":"Fangyin Wei","hidden":false},{"_id":"6847f1f388cb0eba3d0a27d0","name":"Yunhao Ge","hidden":false},{"_id":"6847f1f388cb0eba3d0a27d1","name":"Jinwei Gu","hidden":false},{"_id":"6847f1f388cb0eba3d0a27d2","name":"Ming-Yu Liu","hidden":false},{"_id":"6847f1f388cb0eba3d0a27d3","name":"Abe Davis","hidden":false},{"_id":"6847f1f388cb0eba3d0a27d4","name":"Yifan Ding","hidden":false}],"publishedAt":"2025-05-31T23:03:54.000Z","title":"ArtiScene: Language-Driven Artistic 3D Scene Generation Through Image\n  Intermediary","summary":"Designing 3D scenes is traditionally a challenging task that demands both\nartistic expertise and proficiency with complex software. Recent advances in\ntext-to-3D generation have greatly simplified this process by letting users\ncreate scenes based on simple text descriptions. However, as these methods\ngenerally require extra training or in-context learning, their performance is\noften hindered by the limited availability of high-quality 3D data. In\ncontrast, modern text-to-image models learned from web-scale images can\ngenerate scenes with diverse, reliable spatial layouts and consistent, visually\nappealing styles. Our key insight is that instead of learning directly from 3D\nscenes, we can leverage generated 2D images as an intermediary to guide 3D\nsynthesis. In light of this, we introduce ArtiScene, a training-free automated\npipeline for scene design that integrates the flexibility of free-form\ntext-to-image generation with the diversity and reliability of 2D intermediary\nlayouts.\n  First, we generate 2D images from a scene description, then extract the shape\nand appearance of objects to create 3D models. These models are assembled into\nthe final scene using geometry, position, and pose information derived from the\nsame intermediary image. Being generalizable to a wide range of scenes and\nstyles, ArtiScene outperforms state-of-the-art benchmarks by a large margin in\nlayout and aesthetic quality by quantitative metrics. It also averages a 74.89%\nwinning rate in extensive user studies and 95.07% in GPT-4o evaluation. Project\npage: https://artiscene-cvpr.github.io/","upvotes":0,"discussionId":"6847f1f388cb0eba3d0a27d5","ai_summary":"ArtiScene is an automated pipeline using 2D images as intermediaries to generate high-quality, aesthetically pleasing 3D scenes from text descriptions.","ai_keywords":["text-to-3D generation","text-to-image models","2D images","3D scenes","3D models","scene design","layout","aesthetic quality","user studies","GPT-4o evaluation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-05-31T19:03:54.000Z","title":"ArtiScene: Language-Driven Artistic 3D Scene Generation Through Image\n  Intermediary","summary":"Designing 3D scenes is traditionally a challenging task that demands both\nartistic expertise and proficiency with complex software. Recent advances in\ntext-to-3D generation have greatly simplified this process by letting users\ncreate scenes based on simple text descriptions. However, as these methods\ngenerally require extra training or in-context learning, their performance is\noften hindered by the limited availability of high-quality 3D data. In\ncontrast, modern text-to-image models learned from web-scale images can\ngenerate scenes with diverse, reliable spatial layouts and consistent, visually\nappealing styles. Our key insight is that instead of learning directly from 3D\nscenes, we can leverage generated 2D images as an intermediary to guide 3D\nsynthesis. In light of this, we introduce ArtiScene, a training-free automated\npipeline for scene design that integrates the flexibility of free-form\ntext-to-image generation with the diversity and reliability of 2D intermediary\nlayouts.\n  First, we generate 2D images from a scene description, then extract the shape\nand appearance of objects to create 3D models. These models are assembled into\nthe final scene using geometry, position, and pose information derived from the\nsame intermediary image. Being generalizable to a wide range of scenes and\nstyles, ArtiScene outperforms state-of-the-art benchmarks by a large margin in\nlayout and aesthetic quality by quantitative metrics. It also averages a 74.89%\nwinning rate in extensive user studies and 95.07% in GPT-4o evaluation. Project\npage: https://artiscene-cvpr.github.io/","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2506.00742.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"ArtiScene: Language-Driven Artistic 3D Scene Generation Through Image\n  Intermediary"}],"highlightedSummary":[{"type":"text","text":"Designing 3D scenes is traditionally a challenging task that demands both\nartistic expertise and proficiency with complex software. Recent advances in\ntext-to-3D generation have greatly simplified this process by letting users\ncreate scenes based on simple text descriptions. However, as these methods\ngenerally require extra training or in-context learning, their performance is\noften hindered by the limited availability of high-quality 3D data. "},{"type":"highlight","text":"In\ncontrast, modern text-to-image models learned from web-scale images can\ngenerate scenes with diverse, reliable spatial layouts and consistent, visually\nappealing styles."},{"type":"text","text":" Our key insight is that instead of learning directly from 3D\nscenes, we can leverage generated 2D images as an intermediary to guide 3D\nsynthesis. In light of this, we introduce ArtiScene, a training-free automated\npipeline for scene design that integrates the flexibility of free-form\ntext-to-image generation with the diversity and reliability of 2D intermediary\nlayouts.\n  First, we generate 2D images from a scene description, then extract the shape\nand appearance of objects to create 3D models. These models are assembled into\nthe final scene using geometry, position, and pose information derived from the\nsame intermediary image. Being generalizable to a wide range of scenes and\nstyles, ArtiScene outperforms state-of-the-art benchmarks by a large margin in\nlayout and aesthetic quality by quantitative metrics. It also averages a 74.89%\nwinning rate in extensive user studies and 95.07% in GPT-4o evaluation. Project\npage: https://artiscene-cvpr.github.io/"}]},{"paper":{"id":"2401.10786","authors":[{"_id":"65bb02ea5d2ef0a4b2f736a3","name":"Zuoyue Li","hidden":false},{"_id":"65bb02ea5d2ef0a4b2f736a4","name":"Zhenqiang Li","hidden":false},{"_id":"65bb02ea5d2ef0a4b2f736a5","name":"Zhaopeng Cui","hidden":false},{"_id":"65bb02ea5d2ef0a4b2f736a6","user":{"_id":"67b5fa179782a5e2fd2cb26a","avatarUrl":"/avatars/62c38f29ec641e001eeddf840bea21a0.svg","isPro":false,"fullname":"Marc Pollefeys","user":"mapo1","type":"user","name":"mapo1"},"name":"Marc Pollefeys","status":"claimed_verified","statusLastChangedAt":"2025-02-25T09:43:12.587Z","hidden":false},{"_id":"65bb02ea5d2ef0a4b2f736a7","name":"Martin R. Oswald","hidden":false}],"publishedAt":"2024-01-19T16:15:37.000Z","title":"Sat2Scene: 3D Urban Scene Generation from Satellite Images with\n  Diffusion","summary":"Directly generating scenes from satellite imagery offers exciting\npossibilities for integration into applications like games and map services.\nHowever, challenges arise from significant view changes and scene scale.\nPrevious efforts mainly focused on image or video generation, lacking\nexploration into the adaptability of scene generation for arbitrary views.\nExisting 3D generation works either operate at the object level or are\ndifficult to utilize the geometry obtained from satellite imagery. To overcome\nthese limitations, we propose a novel architecture for direct 3D scene\ngeneration by introducing diffusion models into 3D sparse representations and\ncombining them with neural rendering techniques. Specifically, our approach\ngenerates texture colors at the point level for a given geometry using a 3D\ndiffusion model first, which is then transformed into a scene representation in\na feed-forward manner. The representation can be utilized to render arbitrary\nviews which would excel in both single-frame quality and inter-frame\nconsistency. Experiments in two city-scale datasets show that our model\ndemonstrates proficiency in generating photo-realistic street-view image\nsequences and cross-view urban scenes from satellite imagery.","upvotes":0,"discussionId":"65bb02f05d2ef0a4b2f737f4","githubRepo":"https://github.com/lizuoyue/sat2scene","githubRepoAddedBy":"auto","ai_summary":"A novel architecture combining 3D diffusion models and neural rendering generates photo-realistic scenes from satellite imagery for arbitrary views.","ai_keywords":["diffusion models","3D sparse representations","neural rendering","texture colors","scene representation","feed-forward manner","photo-realistic street-view image sequences","cross-view urban scenes"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":41},"publishedAt":"2024-01-19T11:15:37.000Z","title":"Sat2Scene: 3D Urban Scene Generation from Satellite Images with\n  Diffusion","summary":"Directly generating scenes from satellite imagery offers exciting\npossibilities for integration into applications like games and map services.\nHowever, challenges arise from significant view changes and scene scale.\nPrevious efforts mainly focused on image or video generation, lacking\nexploration into the adaptability of scene generation for arbitrary views.\nExisting 3D generation works either operate at the object level or are\ndifficult to utilize the geometry obtained from satellite imagery. To overcome\nthese limitations, we propose a novel architecture for direct 3D scene\ngeneration by introducing diffusion models into 3D sparse representations and\ncombining them with neural rendering techniques. Specifically, our approach\ngenerates texture colors at the point level for a given geometry using a 3D\ndiffusion model first, which is then transformed into a scene representation in\na feed-forward manner. The representation can be utilized to render arbitrary\nviews which would excel in both single-frame quality and inter-frame\nconsistency. Experiments in two city-scale datasets show that our model\ndemonstrates proficiency in generating photo-realistic street-view image\nsequences and cross-view urban scenes from satellite imagery.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2401.10786.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Sat2Scene: 3D Urban Scene Generation from Satellite Images with\n  Diffusion"}],"highlightedSummary":[{"type":"text","text":"Directly generating scenes from satellite imagery offers exciting\npossibilities for integration into applications like games and map services.\nHowever, challenges arise from significant view changes and scene scale.\nPrevious efforts mainly focused on image or video generation, lacking\nexploration into the adaptability of scene generation for arbitrary views.\nExisting 3D generation works either operate at the object level or are\ndifficult to utilize the geometry obtained from satellite imagery. To overcome\nthese limitations, we propose a novel architecture for direct 3D scene\ngeneration by introducing diffusion models into 3D sparse representations and\ncombining them with neural rendering techniques. Specifically, our approach\ngenerates texture colors at the point level for a given geometry using a 3D\ndiffusion model first, which is then transformed into a scene representation in\na feed-forward manner. The representation can be utilized to render arbitrary\nviews which would excel in both single-frame quality and inter-frame\nconsistency. "},{"type":"highlight","text":"Experiments in two city-scale datasets show that our model\ndemonstrates proficiency in generating photo-realistic street-view image\nsequences and cross-view urban scenes from satellite imagery."}]},{"paper":{"id":"2603.19598","authors":[{"_id":"69c0accd3fd3fef6237a685d","user":{"_id":"6464c4ef92773d5eeb588525","avatarUrl":"/avatars/65fc839453d892016640249cc1acf277.svg","isPro":false,"fullname":"Zhifei Yang","user":"yangzhifei","type":"user","name":"yangzhifei"},"name":"Zhifei Yang","status":"claimed_verified","statusLastChangedAt":"2026-03-23T09:16:10.216Z","hidden":false},{"_id":"69c0accd3fd3fef6237a685e","name":"Guangyao Zhai","hidden":false},{"_id":"69c0accd3fd3fef6237a685f","name":"Keyang Lu","hidden":false},{"_id":"69c0accd3fd3fef6237a6860","name":"YuYang Yin","hidden":false},{"_id":"69c0accd3fd3fef6237a6861","name":"Chao Zhang","hidden":false},{"_id":"69c0accd3fd3fef6237a6862","name":"Zhen Xiao","hidden":false},{"_id":"69c0accd3fd3fef6237a6863","name":"Jieyi Long","hidden":false},{"_id":"69c0accd3fd3fef6237a6864","name":"Nassir Navab","hidden":false},{"_id":"69c0accd3fd3fef6237a6865","name":"Yikai Wang","hidden":false}],"publishedAt":"2026-03-20T03:15:42.000Z","title":"FlowScene: Style-Consistent Indoor Scene Generation with Multimodal Graph Rectified Flow","summary":"Scene generation has extensive industrial applications, demanding both high realism and precise control over geometry and appearance. Language-driven retrieval methods compose plausible scenes from a large object database, but overlook object-level control and often fail to enforce scene-level style coherence. Graph-based formulations offer higher controllability over objects and inform holistic consistency by explicitly modeling relations, yet existing methods struggle to produce high-fidelity textured results, thereby limiting their practical utility. We present FlowScene, a tri-branch scene generative model conditioned on multimodal graphs that collaboratively generates scene layouts, object shapes, and object textures. At its core lies a tight-coupled rectified flow model that exchanges object information during generation, enabling collaborative reasoning across the graph. This enables fine-grained control of objects' shapes, textures, and relations while enforcing scene-level style coherence across structure and appearance. Extensive experiments show that FlowScene outperforms both language-conditioned and graph-conditioned baselines in terms of generation realism, style consistency, and alignment with human preferences.","upvotes":32,"discussionId":"69c0accd3fd3fef6237a6866","ai_summary":"FlowScene is a tri-branch generative model that combines multimodal graph conditioning with rectified flow modeling to produce realistic scenes with controlled geometry, appearance, and stylistic coherence.","ai_keywords":["rectified flow model","multimodal graphs","scene generation","graph-based formulations","object-level control","scene-level style coherence","tri-branch model"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","organization":{"_id":"61dcd8e344f59573371b5cb6","name":"PekingUniversity","fullname":"Peking University","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/noauth/vavgrBsnkSejriUF4lXDE.png"}},"publishedAt":"2026-03-19T23:15:42.000Z","title":"FlowScene: Style-Consistent Indoor Scene Generation with Multimodal Graph Rectified Flow","summary":"Scene generation has extensive industrial applications, demanding both high realism and precise control over geometry and appearance. Language-driven retrieval methods compose plausible scenes from a large object database, but overlook object-level control and often fail to enforce scene-level style coherence. Graph-based formulations offer higher controllability over objects and inform holistic consistency by explicitly modeling relations, yet existing methods struggle to produce high-fidelity textured results, thereby limiting their practical utility. We present FlowScene, a tri-branch scene generative model conditioned on multimodal graphs that collaboratively generates scene layouts, object shapes, and object textures. At its core lies a tight-coupled rectified flow model that exchanges object information during generation, enabling collaborative reasoning across the graph. This enables fine-grained control of objects' shapes, textures, and relations while enforcing scene-level style coherence across structure and appearance. Extensive experiments show that FlowScene outperforms both language-conditioned and graph-conditioned baselines in terms of generation realism, style consistency, and alignment with human preferences.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2603.19598.png","numComments":2,"upvoted":false,"organization":{"_id":"61dcd8e344f59573371b5cb6","name":"PekingUniversity","fullname":"Peking University","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/noauth/vavgrBsnkSejriUF4lXDE.png"},"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"FlowScene: Style-Consistent Indoor Scene Generation with Multimodal Graph Rectified Flow"}],"highlightedSummary":[{"type":"text","text":"Scene generation has extensive industrial applications, demanding both high realism and precise control over geometry and appearance. Language-driven retrieval methods compose plausible scenes from a large object database, but overlook object-level control and often fail to enforce scene-level style coherence. Graph-based formulations offer higher controllability over objects and inform holistic consistency by explicitly modeling relations, yet existing methods struggle to produce high-fidelity textured results, thereby limiting their practical utility. "},{"type":"highlight","text":"We present FlowScene, a tri-branch scene generative model conditioned on multimodal graphs that collaboratively generates scene layouts, object shapes, and object textures."},{"type":"text","text":" At its core lies a tight-coupled rectified flow model that exchanges object information during generation, enabling collaborative reasoning across the graph. This enables fine-grained control of objects' shapes, textures, and relations while enforcing scene-level style coherence across structure and appearance. Extensive experiments show that FlowScene outperforms both language-conditioned and graph-conditioned baselines in terms of generation realism, style consistency, and alignment with human preferences."}]},{"paper":{"id":"2412.08580","authors":[{"_id":"675a6094ca1f2b4cd6b2bfac","user":{"_id":"63a06cebe648d425374f3ea3","avatarUrl":"/avatars/2e15a761aac16bfdc3da46dd5eef1adf.svg","isPro":false,"fullname":"Zejian Li","user":"hyllbd","type":"user","name":"hyllbd"},"name":"Zejian Li","status":"admin_assigned","statusLastChangedAt":"2024-12-12T08:28:01.032Z","hidden":false},{"_id":"675a6094ca1f2b4cd6b2bfad","user":{"_id":"66d1fb162e0412fa2aa4ffb0","avatarUrl":"/avatars/20620dd03d388f0262124f0c3523080d.svg","isPro":false,"fullname":"Chenye Meng","user":"mengcy","type":"user","name":"mengcy"},"name":"Chenye Meng","status":"admin_assigned","statusLastChangedAt":"2024-12-12T08:27:39.200Z","hidden":false},{"_id":"675a6094ca1f2b4cd6b2bfae","user":{"_id":"6552d3f69fa2f7205a1f94a2","avatarUrl":"/avatars/8b857fe89586b25b54864da91ce1b35a.svg","isPro":false,"fullname":"Yize Li","user":"yeezlee","type":"user","name":"yeezlee"},"name":"Yize Li","status":"admin_assigned","statusLastChangedAt":"2024-12-12T08:28:06.791Z","hidden":false},{"_id":"675a6094ca1f2b4cd6b2bfaf","user":{"_id":"64fde4e252e82dd432b74ce9","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64fde4e252e82dd432b74ce9/-CQZbBP7FsPPyawYrsi4z.jpeg","isPro":false,"fullname":"Ling Yang","user":"Lingaaaaaaa","type":"user","name":"Lingaaaaaaa"},"name":"Ling Yang","status":"claimed_verified","statusLastChangedAt":"2025-02-11T15:34:08.086Z","hidden":false},{"_id":"675a6094ca1f2b4cd6b2bfb0","user":{"_id":"63943c882b9483beb473ec25","avatarUrl":"/avatars/abd2aae43e68c34770159c15a01c8297.svg","isPro":false,"fullname":"Shengyuan Zhang","user":"SYZhang0805","type":"user","name":"SYZhang0805"},"name":"Shengyuan Zhang","status":"admin_assigned","statusLastChangedAt":"2024-12-12T08:27:07.929Z","hidden":false},{"_id":"675a6094ca1f2b4cd6b2bfb1","user":{"_id":"6618ae26a7f8bb5a98ce4164","avatarUrl":"/avatars/5656997d49be6fd0f3fd793136098806.svg","isPro":false,"fullname":"JIARUI MA","user":"MAJIARUI","type":"user","name":"MAJIARUI"},"name":"Jiarui Ma","status":"admin_assigned","statusLastChangedAt":"2024-12-12T08:28:15.616Z","hidden":false},{"_id":"675a6094ca1f2b4cd6b2bfb2","name":"Jiayi Li","hidden":false},{"_id":"675a6094ca1f2b4cd6b2bfb3","name":"Guang Yang","hidden":false},{"_id":"675a6094ca1f2b4cd6b2bfb4","name":"Changyuan Yang","hidden":false},{"_id":"675a6094ca1f2b4cd6b2bfb5","name":"Zhiyuan Yang","hidden":false},{"_id":"675a6094ca1f2b4cd6b2bfb6","name":"Jinxiong Chang","hidden":false},{"_id":"675a6094ca1f2b4cd6b2bfb7","user":{"_id":"662a5c0649e03000ed1adab4","avatarUrl":"/avatars/961e4b9bf923923fcd871672d156f50c.svg","isPro":false,"fullname":"Lingyun Sun","user":"slysun","type":"user","name":"slysun"},"name":"Lingyun Sun","status":"admin_assigned","statusLastChangedAt":"2024-12-12T08:25:33.476Z","hidden":false}],"publishedAt":"2024-12-11T17:57:10.000Z","title":"LAION-SG: An Enhanced Large-Scale Dataset for Training Complex\n  Image-Text Models with Structural Annotations","summary":"Recent advances in text-to-image (T2I) generation have shown remarkable\nsuccess in producing high-quality images from text. However, existing T2I\nmodels show decayed performance in compositional image generation involving\nmultiple objects and intricate relationships. We attribute this problem to\nlimitations in existing datasets of image-text pairs, which lack precise\ninter-object relationship annotations with prompts only. To address this\nproblem, we construct LAION-SG, a large-scale dataset with high-quality\nstructural annotations of scene graphs (SG), which precisely describe\nattributes and relationships of multiple objects, effectively representing the\nsemantic structure in complex scenes. Based on LAION-SG, we train a new\nfoundation model SDXL-SG to incorporate structural annotation information into\nthe generation process. Extensive experiments show advanced models trained on\nour LAION-SG boast significant performance improvements in complex scene\ngeneration over models on existing datasets. We also introduce CompSG-Bench, a\nbenchmark that evaluates models on compositional image generation, establishing\na new standard for this domain.","upvotes":45,"discussionId":"675a609cca1f2b4cd6b2c12f","projectPage":"https://huggingface.co/datasets/mengcy/LAION-SG","githubRepo":"https://github.com/mengcye/LAION-SG","githubRepoAddedBy":"user","ai_summary":"A new dataset with structural scene graph annotations improves text-to-image generation performance on complex scenes, benchmarked by CompSG-Bench.","ai_keywords":["text-to-image (T2I)","compositional image generation","LAION-SG","scene graphs (SG)","SDXL-SG","CompSG-Bench"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":58},"publishedAt":"2024-12-11T12:57:10.000Z","title":"LAION-SG: An Enhanced Large-Scale Dataset for Training Complex\n  Image-Text Models with Structural Annotations","summary":"Recent advances in text-to-image (T2I) generation have shown remarkable\nsuccess in producing high-quality images from text. However, existing T2I\nmodels show decayed performance in compositional image generation involving\nmultiple objects and intricate relationships. We attribute this problem to\nlimitations in existing datasets of image-text pairs, which lack precise\ninter-object relationship annotations with prompts only. To address this\nproblem, we construct LAION-SG, a large-scale dataset with high-quality\nstructural annotations of scene graphs (SG), which precisely describe\nattributes and relationships of multiple objects, effectively representing the\nsemantic structure in complex scenes. Based on LAION-SG, we train a new\nfoundation model SDXL-SG to incorporate structural annotation information into\nthe generation process. Extensive experiments show advanced models trained on\nour LAION-SG boast significant performance improvements in complex scene\ngeneration over models on existing datasets. We also introduce CompSG-Bench, a\nbenchmark that evaluates models on compositional image generation, establishing\na new standard for this domain.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2412.08580.png","numComments":3,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"LAION-SG: An Enhanced Large-Scale Dataset for Training Complex\n  Image-Text Models with Structural Annotations"}],"highlightedSummary":[{"type":"text","text":"Recent advances in text-to-image (T2I) generation have shown remarkable\nsuccess in producing high-quality images from text. "},{"type":"highlight","text":"However, existing T2I\nmodels show decayed performance in compositional image generation involving\nmultiple objects and intricate relationships."},{"type":"text","text":" We attribute this problem to\nlimitations in existing datasets of image-text pairs, which lack precise\ninter-object relationship annotations with prompts only. To address this\nproblem, we construct LAION-SG, a large-scale dataset with high-quality\nstructural annotations of scene graphs (SG), which precisely describe\nattributes and relationships of multiple objects, effectively representing the\nsemantic structure in complex scenes. Based on LAION-SG, we train a new\nfoundation model SDXL-SG to incorporate structural annotation information into\nthe generation process. Extensive experiments show advanced models trained on\nour LAION-SG boast significant performance improvements in complex scene\ngeneration over models on existing datasets. We also introduce CompSG-Bench, a\nbenchmark that evaluates models on compositional image generation, establishing\na new standard for this domain."}]},{"paper":{"id":"2306.08068","authors":[{"_id":"648bf1bc335645aa03f03965","name":"Allan Jabri","hidden":false},{"_id":"648bf1bc335645aa03f03966","name":"Sjoerd van Steenkiste","hidden":false},{"_id":"648bf1bc335645aa03f03967","name":"Emiel Hoogeboom","hidden":false},{"_id":"648bf1bc335645aa03f03968","name":"Mehdi S. M. Sajjadi","hidden":false},{"_id":"648bf1bc335645aa03f03969","user":{"_id":"6413572b6cd62eb3ba2024e9","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6413572b6cd62eb3ba2024e9/UCtMdeV8_N6w-SBnTDXBy.jpeg","isPro":false,"fullname":"Thomas Kipf","user":"tkipf","type":"user","name":"tkipf"},"name":"Thomas Kipf","status":"claimed_verified","statusLastChangedAt":"2023-06-16T10:52:05.911Z","hidden":false}],"publishedAt":"2023-06-13T18:32:35.000Z","title":"DORSal: Diffusion for Object-centric Representations of Scenes\n  et al.","summary":"Recent progress in 3D scene understanding enables scalable learning of\nrepresentations across large datasets of diverse scenes. As a consequence,\ngeneralization to unseen scenes and objects, rendering novel views from just a\nsingle or a handful of input images, and controllable scene generation that\nsupports editing, is now possible. However, training jointly on a large number\nof scenes typically compromises rendering quality when compared to single-scene\noptimized models such as NeRFs. In this paper, we leverage recent progress in\ndiffusion models to equip 3D scene representation learning models with the\nability to render high-fidelity novel views, while retaining benefits such as\nobject-level scene editing to a large degree. In particular, we propose DORSal,\nwhich adapts a video diffusion architecture for 3D scene generation conditioned\non object-centric slot-based representations of scenes. On both complex\nsynthetic multi-object scenes and on the real-world large-scale Street View\ndataset, we show that DORSal enables scalable neural rendering of 3D scenes\nwith object-level editing and improves upon existing approaches.","upvotes":6,"discussionId":"648bf1be335645aa03f039ed","ai_summary":"A video diffusion architecture adapted for 3D scene generation improves rendering quality and scalability while maintaining object-level editing capabilities.","ai_keywords":["diffusion models","DORSal","video diffusion architecture","3D scene generation","object-centric slot-based representations","neural rendering"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2023-06-13T14:32:35.000Z","title":"DORSal: Diffusion for Object-centric Representations of Scenes\n  et al.","summary":"Recent progress in 3D scene understanding enables scalable learning of\nrepresentations across large datasets of diverse scenes. As a consequence,\ngeneralization to unseen scenes and objects, rendering novel views from just a\nsingle or a handful of input images, and controllable scene generation that\nsupports editing, is now possible. However, training jointly on a large number\nof scenes typically compromises rendering quality when compared to single-scene\noptimized models such as NeRFs. In this paper, we leverage recent progress in\ndiffusion models to equip 3D scene representation learning models with the\nability to render high-fidelity novel views, while retaining benefits such as\nobject-level scene editing to a large degree. In particular, we propose DORSal,\nwhich adapts a video diffusion architecture for 3D scene generation conditioned\non object-centric slot-based representations of scenes. On both complex\nsynthetic multi-object scenes and on the real-world large-scale Street View\ndataset, we show that DORSal enables scalable neural rendering of 3D scenes\nwith object-level editing and improves upon existing approaches.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2306.08068.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"DORSal: Diffusion for Object-centric Representations of Scenes\n  et al."}],"highlightedSummary":[{"type":"text","text":"Recent progress in 3D scene understanding enables scalable learning of\nrepresentations across large datasets of diverse scenes. "},{"type":"highlight","text":"As a consequence,\ngeneralization to unseen scenes and objects, rendering novel views from just a\nsingle or a handful of input images, and controllable scene generation that\nsupports editing, is now possible."},{"type":"text","text":" However, training jointly on a large number\nof scenes typically compromises rendering quality when compared to single-scene\noptimized models such as NeRFs. In this paper, we leverage recent progress in\ndiffusion models to equip 3D scene representation learning models with the\nability to render high-fidelity novel views, while retaining benefits such as\nobject-level scene editing to a large degree. In particular, we propose DORSal,\nwhich adapts a video diffusion architecture for 3D scene generation conditioned\non object-centric slot-based representations of scenes. On both complex\nsynthetic multi-object scenes and on the real-world large-scale Street View\ndataset, we show that DORSal enables scalable neural rendering of 3D scenes\nwith object-level editing and improves upon existing approaches."}]},{"paper":{"id":"2304.08463","authors":[{"_id":"6a8fd26f3bd48bb654ea6924","name":"Yilun Du","hidden":false},{"_id":"6a8fd26f3bd48bb654ea6925","name":"Cameron Smith","hidden":false},{"_id":"6a8fd26f3bd48bb654ea6926","name":"Ayush Tewari","hidden":false},{"_id":"6a8fd26f3bd48bb654ea6927","name":"Vincent Sitzmann","hidden":false}],"publishedAt":"2023-04-17T00:00:00.000Z","title":"Learning to Render Novel Views from Wide-Baseline Stereo Pairs","summary":"We introduce a method for novel view synthesis given only a single wide-baseline stereo image pair. In this challenging regime, 3D scene points are regularly observed only once, requiring prior-based reconstruction of scene geometry and appearance. We find that existing approaches to novel view synthesis from sparse observations fail due to recovering incorrect 3D geometry and due to the high cost of differentiable rendering that precludes their scaling to large-scale training. We take a step towards resolving these shortcomings by formulating a multi-view transformer encoder, proposing an efficient, image-space epipolar line sampling scheme to assemble image features for a target ray, and a lightweight cross-attention-based renderer. Our contributions enable training of our method on a large-scale real-world dataset of indoor and outdoor scenes. We demonstrate that our method learns powerful multi-view geometry priors while reducing the rendering time. We conduct extensive comparisons on held-out test scenes across two real-world datasets, significantly outperforming prior work on novel view synthesis from sparse image observations and achieving multi-view-consistent novel view synthesis.","upvotes":0,"discussionId":"6a8fd2703bd48bb654ea6928","ai_summary":"A multi-view transformer with efficient epipolar sampling and lightweight cross-attention rendering enables scalable training for high-quality novel view synthesis from sparse wide-baseline stereo pairs.","ai_keywords":["multi-view transformer","epipolar line sampling","cross-attention-based renderer","novel view synthesis","wide-baseline stereo","differentiable rendering"],"ai_summary_model":"thinkingmachines/Inkling-Small"},"publishedAt":"2023-04-16T20:00:00.000Z","title":"Learning to Render Novel Views from Wide-Baseline Stereo Pairs","summary":"We introduce a method for novel view synthesis given only a single wide-baseline stereo image pair. In this challenging regime, 3D scene points are regularly observed only once, requiring prior-based reconstruction of scene geometry and appearance. We find that existing approaches to novel view synthesis from sparse observations fail due to recovering incorrect 3D geometry and due to the high cost of differentiable rendering that precludes their scaling to large-scale training. We take a step towards resolving these shortcomings by formulating a multi-view transformer encoder, proposing an efficient, image-space epipolar line sampling scheme to assemble image features for a target ray, and a lightweight cross-attention-based renderer. Our contributions enable training of our method on a large-scale real-world dataset of indoor and outdoor scenes. We demonstrate that our method learns powerful multi-view geometry priors while reducing the rendering time. We conduct extensive comparisons on held-out test scenes across two real-world datasets, significantly outperforming prior work on novel view synthesis from sparse image observations and achieving multi-view-consistent novel view synthesis.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2304.08463.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Learning to Render Novel Views from Wide-Baseline Stereo Pairs"}],"highlightedSummary":[{"type":"text","text":"We introduce a method for novel view synthesis given only a single wide-baseline stereo image pair. "},{"type":"highlight","text":"In this challenging regime, 3D scene points are regularly observed only once, requiring prior-based reconstruction of scene geometry and appearance."},{"type":"text","text":" We find that existing approaches to novel view synthesis from sparse observations fail due to recovering incorrect 3D geometry and due to the high cost of differentiable rendering that precludes their scaling to large-scale training. We take a step towards resolving these shortcomings by formulating a multi-view transformer encoder, proposing an efficient, image-space epipolar line sampling scheme to assemble image features for a target ray, and a lightweight cross-attention-based renderer. Our contributions enable training of our method on a large-scale real-world dataset of indoor and outdoor scenes. We demonstrate that our method learns powerful multi-view geometry priors while reducing the rendering time. We conduct extensive comparisons on held-out test scenes across two real-world datasets, significantly outperforming prior work on novel view synthesis from sparse image observations and achieving multi-view-consistent novel view synthesis."}]},{"paper":{"id":"2604.01043","authors":[{"_id":"69d3d4922c523d36431faea8","user":{"_id":"689a9f1ae4aff5e9a73c4cd4","avatarUrl":"/avatars/193557aa81cbddca2b5c12fb2b11227b.svg","isPro":false,"fullname":"Fengyuan Yang","user":"MartaYang007","type":"user","name":"MartaYang007"},"name":"Fengyuan Yang","status":"claimed_verified","statusLastChangedAt":"2026-04-07T08:37:44.047Z","hidden":false},{"_id":"69d3d4922c523d36431faea9","user":{"_id":"64d607488f84a7738d78a4ff","avatarUrl":"/avatars/2a7b57fb15bc4ae457b5e18f2da07890.svg","isPro":false,"fullname":"LUYING","user":"hlyyyyy","type":"user","name":"hlyyyyy"},"name":"Luying Huang","status":"claimed_verified","statusLastChangedAt":"2026-04-07T08:37:48.187Z","hidden":false},{"_id":"69d3d4922c523d36431faeaa","name":"Jiazhi Guan","hidden":false},{"_id":"69d3d4922c523d36431faeab","name":"Quanwei Yang","hidden":false},{"_id":"69d3d4922c523d36431faeac","name":"Dongwei Pan","hidden":false},{"_id":"69d3d4922c523d36431faead","name":"Jianglin Fu","hidden":false},{"_id":"69d3d4922c523d36431faeae","name":"Haocheng Feng","hidden":false},{"_id":"69d3d4922c523d36431faeaf","name":"Wei He","hidden":false},{"_id":"69d3d4922c523d36431faeb0","name":"Kaisiyuan Wang","hidden":false},{"_id":"69d3d4922c523d36431faeb1","name":"Hang Zhou","hidden":false},{"_id":"69d3d4922c523d36431faeb2","name":"Angela Yao","hidden":false}],"publishedAt":"2026-04-01T00:00:00.000Z","title":"ONE-SHOT: Compositional Human-Environment Video Synthesis via Spatial-Decoupled Motion Injection and Hybrid Context Integration","summary":"Recent advances in Video Foundation Models (VFMs) have revolutionized human-centric video synthesis, yet fine-grained and independent editing of subjects and scenes remains a critical challenge. Recent attempts to incorporate richer environment control through rigid 3D geometric compositions often encounter a stark trade-off between precise control and generative flexibility. Furthermore, the heavy 3D pre-processing still limits practical scalability. In this paper, we propose ONE-SHOT, a parameter-efficient framework for compositional human-environment video generation. Our key insight is to factorize the generative process into disentangled signals. Specifically, we introduce a canonical-space injection mechanism that decouples human dynamics from environmental cues via cross-attention. We also propose Dynamic-Grounded-RoPE, a novel positional embedding strategy that establishes spatial correspondences between disparate spatial domains without any heuristic 3D alignments. To support long-horizon synthesis, we introduce a Hybrid Context Integration mechanism to maintain subject and scene consistency across minute-level generations. Experiments demonstrate that our method significantly outperforms state-of-the-art methods, offering superior structural control and creative diversity for video synthesis. Our project has been available on: https://martayang.github.io/ONE-SHOT/.","upvotes":9,"discussionId":"69d3d4922c523d36431faeb3","projectPage":"https://martayang.github.io/ONE-SHOT/","ai_summary":"ONE-SHOT enables compositional human-environment video generation through disentangled signals, dynamic positional embeddings, and hybrid context integration for improved control and diversity.","ai_keywords":["Video Foundation Models","parameter-efficient framework","canonical-space injection mechanism","cross-attention","Dynamic-Grounded-RoPE","positional embedding","Hybrid Context Integration","long-horizon synthesis","disentangled signals"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","organization":{"_id":"6508ab2b349930913196378b","name":"NationalUniversityofSingapore","fullname":"National University of Singapore","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/630ca0817dacb93b33506ce7/ZYUmpSMsa5Whihw3me2Bw.png"}},"publishedAt":"2026-03-31T20:00:00.000Z","title":"ONE-SHOT: Compositional Human-Environment Video Synthesis via Spatial-Decoupled Motion Injection and Hybrid Context Integration","summary":"Recent advances in Video Foundation Models (VFMs) have revolutionized human-centric video synthesis, yet fine-grained and independent editing of subjects and scenes remains a critical challenge. Recent attempts to incorporate richer environment control through rigid 3D geometric compositions often encounter a stark trade-off between precise control and generative flexibility. Furthermore, the heavy 3D pre-processing still limits practical scalability. In this paper, we propose ONE-SHOT, a parameter-efficient framework for compositional human-environment video generation. Our key insight is to factorize the generative process into disentangled signals. Specifically, we introduce a canonical-space injection mechanism that decouples human dynamics from environmental cues via cross-attention. We also propose Dynamic-Grounded-RoPE, a novel positional embedding strategy that establishes spatial correspondences between disparate spatial domains without any heuristic 3D alignments. To support long-horizon synthesis, we introduce a Hybrid Context Integration mechanism to maintain subject and scene consistency across minute-level generations. Experiments demonstrate that our method significantly outperforms state-of-the-art methods, offering superior structural control and creative diversity for video synthesis. Our project has been available on: https://martayang.github.io/ONE-SHOT/.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2604.01043.png","numComments":2,"upvoted":false,"organization":{"_id":"6508ab2b349930913196378b","name":"NationalUniversityofSingapore","fullname":"National University of Singapore","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/630ca0817dacb93b33506ce7/ZYUmpSMsa5Whihw3me2Bw.png"},"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"ONE-SHOT: Compositional Human-Environment Video Synthesis via Spatial-Decoupled Motion Injection and Hybrid Context Integration"}],"highlightedSummary":[{"type":"text","text":"Recent advances in Video Foundation Models (VFMs) have revolutionized human-centric video synthesis, yet fine-grained and independent editing of subjects and scenes remains a critical challenge. Recent attempts to incorporate richer environment control through rigid 3D geometric compositions often encounter a stark trade-off between precise control and generative flexibility. "},{"type":"highlight","text":"Furthermore, the heavy 3D pre-processing still limits practical scalability."},{"type":"text","text":" In this paper, we propose ONE-SHOT, a parameter-efficient framework for compositional human-environment video generation. Our key insight is to factorize the generative process into disentangled signals. Specifically, we introduce a canonical-space injection mechanism that decouples human dynamics from environmental cues via cross-attention. We also propose Dynamic-Grounded-RoPE, a novel positional embedding strategy that establishes spatial correspondences between disparate spatial domains without any heuristic 3D alignments. To support long-horizon synthesis, we introduce a Hybrid Context Integration mechanism to maintain subject and scene consistency across minute-level generations. Experiments demonstrate that our method significantly outperforms state-of-the-art methods, offering superior structural control and creative diversity for video synthesis. Our project has been available on: https://martayang.github.io/ONE-SHOT/."}]},{"paper":{"id":"2501.08983","authors":[{"_id":"678897a4d42825b51c19d65a","user":{"_id":"63f47b5321eb234ab739e91a","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/63f47b5321eb234ab739e91a/vWfFNVtMkHl8gieha5PPd.jpeg","isPro":false,"fullname":"Haozhe Xie","user":"hzxie","type":"user","name":"hzxie"},"name":"Haozhe Xie","status":"admin_assigned","statusLastChangedAt":"2025-01-16T08:50:10.101Z","hidden":false},{"_id":"678897a4d42825b51c19d65b","user":{"_id":"62fc8cf7ee999004b5a8b982","avatarUrl":"/avatars/6c5dda9e58747054a989f077a078f3dc.svg","isPro":false,"fullname":"Zhaoxi Chen","user":"FrozenBurning","type":"user","name":"FrozenBurning"},"name":"Zhaoxi Chen","status":"admin_assigned","statusLastChangedAt":"2025-01-16T08:50:16.000Z","hidden":false},{"_id":"678897a4d42825b51c19d65c","user":{"_id":"623c530013a63ea865f96c8e","avatarUrl":"/avatars/164455a1a94f92b71733fc778c21bd89.svg","isPro":false,"fullname":"Fangzhou Hong","user":"hongfz16","type":"user","name":"hongfz16"},"name":"Fangzhou Hong","status":"admin_assigned","statusLastChangedAt":"2025-01-16T08:51:11.083Z","hidden":false},{"_id":"678897a4d42825b51c19d65d","user":{"_id":"62ab1ac1d48b4d8b048a3473","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1656826685333-62ab1ac1d48b4d8b048a3473.png","isPro":false,"fullname":"Ziwei Liu","user":"liuziwei7","type":"user","name":"liuziwei7"},"name":"Ziwei Liu","status":"admin_assigned","statusLastChangedAt":"2025-01-16T08:50:34.647Z","hidden":false}],"publishedAt":"2025-01-15T17:59:56.000Z","title":"CityDreamer4D: Compositional Generative Model of Unbounded 4D Cities","summary":"3D scene generation has garnered growing attention in recent years and has\nmade significant progress. Generating 4D cities is more challenging than 3D\nscenes due to the presence of structurally complex, visually diverse objects\nlike buildings and vehicles, and heightened human sensitivity to distortions in\nurban environments. To tackle these issues, we propose CityDreamer4D, a\ncompositional generative model specifically tailored for generating unbounded\n4D cities. Our main insights are 1) 4D city generation should separate dynamic\nobjects (e.g., vehicles) from static scenes (e.g., buildings and roads), and 2)\nall objects in the 4D scene should be composed of different types of neural\nfields for buildings, vehicles, and background stuff. Specifically, we propose\nTraffic Scenario Generator and Unbounded Layout Generator to produce dynamic\ntraffic scenarios and static city layouts using a highly compact BEV\nrepresentation. Objects in 4D cities are generated by combining stuff-oriented\nand instance-oriented neural fields for background stuff, buildings, and\nvehicles. To suit the distinct characteristics of background stuff and\ninstances, the neural fields employ customized generative hash grids and\nperiodic positional embeddings as scene parameterizations. Furthermore, we\noffer a comprehensive suite of datasets for city generation, including OSM,\nGoogleEarth, and CityTopia. The OSM dataset provides a variety of real-world\ncity layouts, while the Google Earth and CityTopia datasets deliver\nlarge-scale, high-quality city imagery complete with 3D instance annotations.\nLeveraging its compositional design, CityDreamer4D supports a range of\ndownstream applications, such as instance editing, city stylization, and urban\nsimulation, while delivering state-of-the-art performance in generating\nrealistic 4D cities.","upvotes":22,"discussionId":"678897a7d42825b51c19d702","projectPage":"https://haozhexie.com/project/city-dreamer-4d","githubRepo":"https://github.com/hzxie/CityDreamer4D","githubRepoAddedBy":"user","ai_summary":"CityDreamer4D generates realistic 4D cities by separating dynamic and static elements using customized neural fields and compositional generative models.","ai_keywords":["compositional generative model","Traffic Scenario Generator","Unbounded Layout Generator","BEV representation","neural fields","generative hash grids","periodic positional embeddings","scene parameterizations"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":151},"publishedAt":"2025-01-15T12:59:56.000Z","title":"CityDreamer4D: Compositional Generative Model of Unbounded 4D Cities","summary":"3D scene generation has garnered growing attention in recent years and has\nmade significant progress. Generating 4D cities is more challenging than 3D\nscenes due to the presence of structurally complex, visually diverse objects\nlike buildings and vehicles, and heightened human sensitivity to distortions in\nurban environments. To tackle these issues, we propose CityDreamer4D, a\ncompositional generative model specifically tailored for generating unbounded\n4D cities. Our main insights are 1) 4D city generation should separate dynamic\nobjects (e.g., vehicles) from static scenes (e.g., buildings and roads), and 2)\nall objects in the 4D scene should be composed of different types of neural\nfields for buildings, vehicles, and background stuff. Specifically, we propose\nTraffic Scenario Generator and Unbounded Layout Generator to produce dynamic\ntraffic scenarios and static city layouts using a highly compact BEV\nrepresentation. Objects in 4D cities are generated by combining stuff-oriented\nand instance-oriented neural fields for background stuff, buildings, and\nvehicles. To suit the distinct characteristics of background stuff and\ninstances, the neural fields employ customized generative hash grids and\nperiodic positional embeddings as scene parameterizations. Furthermore, we\noffer a comprehensive suite of datasets for city generation, including OSM,\nGoogleEarth, and CityTopia. The OSM dataset provides a variety of real-world\ncity layouts, while the Google Earth and CityTopia datasets deliver\nlarge-scale, high-quality city imagery complete with 3D instance annotations.\nLeveraging its compositional design, CityDreamer4D supports a range of\ndownstream applications, such as instance editing, city stylization, and urban\nsimulation, while delivering state-of-the-art performance in generating\nrealistic 4D cities.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2501.08983.png","numComments":2,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"CityDreamer4D: Compositional Generative Model of Unbounded 4D Cities"}],"highlightedSummary":[{"type":"highlight","text":"3D scene generation has garnered growing attention in recent years and has\nmade significant progress."},{"type":"text","text":" Generating 4D cities is more challenging than 3D\nscenes due to the presence of structurally complex, visually diverse objects\nlike buildings and vehicles, and heightened human sensitivity to distortions in\nurban environments. To tackle these issues, we propose CityDreamer4D, a\ncompositional generative model specifically tailored for generating unbounded\n4D cities. Our main insights are 1) 4D city generation should separate dynamic\nobjects (e.g., vehicles) from static scenes (e.g., buildings and roads), and 2)\nall objects in the 4D scene should be composed of different types of neural\nfields for buildings, vehicles, and background stuff. Specifically, we propose\nTraffic Scenario Generator and Unbounded Layout Generator to produce dynamic\ntraffic scenarios and static city layouts using a highly compact BEV\nrepresentation. Objects in 4D cities are generated by combining stuff-oriented\nand instance-oriented neural fields for background stuff, buildings, and\nvehicles. To suit the distinct characteristics of background stuff and\ninstances, the neural fields employ customized generative hash grids and\nperiodic positional embeddings as scene parameterizations. Furthermore, we\noffer a comprehensive suite of datasets for city generation, including OSM,\nGoogleEarth, and CityTopia. The OSM dataset provides a variety of real-world\ncity layouts, while the Google Earth and CityTopia datasets deliver\nlarge-scale, high-quality city imagery complete with 3D instance annotations.\nLeveraging its compositional design, CityDreamer4D supports a range of\ndownstream applications, such as instance editing, city stylization, and urban\nsimulation, while delivering state-of-the-art performance in generating\nrealistic 4D cities."}]},{"paper":{"id":"2404.06903","authors":[{"_id":"661743070795b7cd10a2ade6","user":{"_id":"642a276516d4d8293c9a47e8","avatarUrl":"/avatars/80e6db8bc2544f3486b11b57858a8692.svg","isPro":false,"fullname":"Shijie Zhou","user":"shijiezhou","type":"user","name":"shijiezhou"},"name":"Shijie Zhou","status":"claimed_verified","statusLastChangedAt":"2024-04-11T06:42:05.581Z","hidden":false},{"_id":"661743070795b7cd10a2ade7","user":{"_id":"6526386e1c6a09292d8d0a22","avatarUrl":"/avatars/471de830de2d775d35368678c1579f87.svg","isPro":false,"fullname":"fan","user":"Fanzhiwen","type":"user","name":"Fanzhiwen"},"name":"Zhiwen Fan","status":"admin_assigned","statusLastChangedAt":"2024-04-18T13:24:16.216Z","hidden":false},{"_id":"661743070795b7cd10a2ade8","user":{"_id":"62f687a0c58915315c4ff75d","avatarUrl":"/avatars/b657180c7666735062782edd4f6a69c9.svg","isPro":false,"fullname":"Dejia Xu","user":"ir1d","type":"user","name":"ir1d"},"name":"Dejia Xu","status":"admin_assigned","statusLastChangedAt":"2024-04-18T13:24:25.149Z","hidden":false},{"_id":"661743070795b7cd10a2ade9","name":"Haoran Chang","hidden":false},{"_id":"661743070795b7cd10a2adea","user":{"_id":"64e2ac52cf8c236f30176499","avatarUrl":"/avatars/0438be7e7b502a2892444249b49b92fe.svg","isPro":false,"fullname":"Pradyumna Chari","user":"pradyumnachari","type":"user","name":"pradyumnachari"},"name":"Pradyumna Chari","status":"admin_assigned","statusLastChangedAt":"2024-04-18T13:24:42.880Z","hidden":false},{"_id":"661743070795b7cd10a2adeb","name":"Tejas Bharadwaj","hidden":false},{"_id":"661743070795b7cd10a2adec","name":"Suya You","hidden":false},{"_id":"661743070795b7cd10a2aded","name":"Zhangyang Wang","hidden":false},{"_id":"661743070795b7cd10a2adee","name":"Achuta Kadambi","hidden":false}],"publishedAt":"2024-04-10T10:46:59.000Z","title":"DreamScene360: Unconstrained Text-to-3D Scene Generation with Panoramic\n  Gaussian Splatting","summary":"The increasing demand for virtual reality applications has highlighted the\nsignificance of crafting immersive 3D assets. We present a text-to-3D\n360^{circ} scene generation pipeline that facilitates the creation of\ncomprehensive 360^{circ} scenes for in-the-wild environments in a matter of\nminutes. Our approach utilizes the generative power of a 2D diffusion model and\nprompt self-refinement to create a high-quality and globally coherent panoramic\nimage. This image acts as a preliminary \"flat\" (2D) scene representation.\nSubsequently, it is lifted into 3D Gaussians, employing splatting techniques to\nenable real-time exploration. To produce consistent 3D geometry, our pipeline\nconstructs a spatially coherent structure by aligning the 2D monocular depth\ninto a globally optimized point cloud. This point cloud serves as the initial\nstate for the centroids of 3D Gaussians. In order to address invisible issues\ninherent in single-view inputs, we impose semantic and geometric constraints on\nboth synthesized and input camera views as regularizations. These guide the\noptimization of Gaussians, aiding in the reconstruction of unseen regions. In\nsummary, our method offers a globally consistent 3D scene within a\n360^{circ} perspective, providing an enhanced immersive experience over\nexisting techniques. Project website at: http://dreamscene360.github.io/","upvotes":21,"discussionId":"6617430d0795b7cd10a2af29","ai_summary":"A text-to-3D 360° scene generation pipeline creates immersive environments using a 2D diffusion model and lifts flat panoramas into 3D Gaussians with semantic and geometric constraints.","ai_keywords":["diffusion model","prompt self-refinement","panoramic image","3D Gaussians","splatting techniques","2D monocular depth","point cloud","semantic constraints","geometric constraints"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-04-10T06:46:59.000Z","title":"DreamScene360: Unconstrained Text-to-3D Scene Generation with Panoramic\n  Gaussian Splatting","summary":"The increasing demand for virtual reality applications has highlighted the\nsignificance of crafting immersive 3D assets. We present a text-to-3D\n360^{circ} scene generation pipeline that facilitates the creation of\ncomprehensive 360^{circ} scenes for in-the-wild environments in a matter of\nminutes. Our approach utilizes the generative power of a 2D diffusion model and\nprompt self-refinement to create a high-quality and globally coherent panoramic\nimage. This image acts as a preliminary \"flat\" (2D) scene representation.\nSubsequently, it is lifted into 3D Gaussians, employing splatting techniques to\nenable real-time exploration. To produce consistent 3D geometry, our pipeline\nconstructs a spatially coherent structure by aligning the 2D monocular depth\ninto a globally optimized point cloud. This point cloud serves as the initial\nstate for the centroids of 3D Gaussians. In order to address invisible issues\ninherent in single-view inputs, we impose semantic and geometric constraints on\nboth synthesized and input camera views as regularizations. These guide the\noptimization of Gaussians, aiding in the reconstruction of unseen regions. In\nsummary, our method offers a globally consistent 3D scene within a\n360^{circ} perspective, providing an enhanced immersive experience over\nexisting techniques. Project website at: http://dreamscene360.github.io/","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2404.06903.png","numComments":3,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"DreamScene360: Unconstrained Text-to-3D Scene Generation with Panoramic\n  Gaussian Splatting"}],"highlightedSummary":[{"type":"text","text":"The increasing demand for virtual reality applications has highlighted the\nsignificance of crafting immersive 3D assets. We present a text-to-3D\n360^{circ} scene generation pipeline that facilitates the creation of\ncomprehensive 360^{circ} scenes for in-the-wild environments in a matter of\nminutes. Our approach utilizes the generative power of a 2D diffusion model and\nprompt self-refinement to create a high-quality and globally coherent panoramic\nimage. This image acts as a preliminary \"flat\" (2D) scene representation.\nSubsequently, it is lifted into 3D Gaussians, employing splatting techniques to\nenable real-time exploration. To produce consistent 3D geometry, our pipeline\nconstructs a spatially coherent structure by aligning the 2D monocular depth\ninto a globally optimized point cloud. This point cloud serves as the initial\nstate for the centroids of 3D Gaussians. In order to address invisible issues\ninherent in single-view inputs, we impose semantic and geometric constraints on\nboth synthesized and input camera views as regularizations. These guide the\noptimization of Gaussians, aiding in the reconstruction of unseen regions. In\nsummary, our method offers a globally consistent 3D scene within a\n360^{circ} perspective, providing an enhanced immersive experience over\nexisting techniques. Project website at: http://dreamscene360.github.io/"}]},{"paper":{"id":"2607.04540","authors":[{"_id":"6a4e76a848d70828b718dcea","name":"Geonung Kim","hidden":false},{"_id":"6a4e76a848d70828b718dceb","name":"Jeongeun Park","hidden":false},{"_id":"6a4e76a848d70828b718dcec","name":"Nuri Ryu","hidden":false},{"_id":"6a4e76a848d70828b718dced","name":"Di Liu","hidden":false},{"_id":"6a4e76a848d70828b718dcee","name":"Sunghyun Cho","hidden":false}],"publishedAt":"2026-07-05T00:00:00.000Z","title":"SceneFrom3D: Geometry-Conditioned Outdoor 3D Scene Generation via View Scheduling with Object-Level Control","summary":"Geometry-conditioned 3D scene generation enables the creation of 3D environments from user-provided geometry, offering direct control over scene structure and object layout. To generate such 3D scenes, current methods commonly adopt a three-stage design that first defines a view schedule, then synthesizes multi-view observations along the scheduled views, and finally reconstructs a 3D representation from the generated images. However, defining the view schedule becomes a major bottleneck for outdoor scenes, where large, unstructured, and unbounded geometry makes it difficult to obtain views that provide sufficient coverage while supporting stable generation. To address this bottleneck, we present SceneFrom3D, a framework that automatically schedules views from outdoor input geometries. SceneFrom3D constructs a directed generation graph whose nodes represent anchor views and whose edges represent interpolation trajectories, defining which views to synthesize, which view pairs to interpolate, and in which order generation should proceed. Beyond automatic view scheduling, SceneFrom3D further improves controllability through object-level conditioning, assigning each object an identity image for appearance guidance and a geometry-adherence parameter for region-wise control over the input geometry. Experiments demonstrate that SceneFrom3D achieves state-of-the-art geometry-conditioned outdoor 3D scene generation, producing high-quality scenes with controllable object appearance and geometry adherence.","upvotes":4,"discussionId":"6a4e76a848d70828b718dcef","projectPage":"https://kimgeonung.github.io/SceneFrom3D/","githubRepo":"https://github.com/KIMGEONUNG/SceneFrom3D","githubRepoAddedBy":"user","ai_summary":"SceneFrom3D generates 3D outdoor scenes by automatically scheduling views from input geometry and controlling object appearance and geometry adherence through identity images and geometry-adherence parameters.","ai_keywords":["3D scene generation","view schedule","multi-view observations","3D reconstruction","directed generation graph","anchor views","interpolation trajectories","object-level conditioning","identity image","geometry-adherence parameter"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":12,"organization":{"_id":"6a1d7b2c2c850ce78fcd590a","name":"postech-cglab","fullname":"POSTECH Computer Graphics Lab","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/65ad0940d63812c33e033124/v90h7lJrY0VAWqHLkMcxM.png"}},"publishedAt":"2026-07-04T20:00:00.000Z","title":"SceneFrom3D: Geometry-Conditioned Outdoor 3D Scene Generation via View Scheduling with Object-Level Control","summary":"Geometry-conditioned 3D scene generation enables the creation of 3D environments from user-provided geometry, offering direct control over scene structure and object layout. To generate such 3D scenes, current methods commonly adopt a three-stage design that first defines a view schedule, then synthesizes multi-view observations along the scheduled views, and finally reconstructs a 3D representation from the generated images. However, defining the view schedule becomes a major bottleneck for outdoor scenes, where large, unstructured, and unbounded geometry makes it difficult to obtain views that provide sufficient coverage while supporting stable generation. To address this bottleneck, we present SceneFrom3D, a framework that automatically schedules views from outdoor input geometries. SceneFrom3D constructs a directed generation graph whose nodes represent anchor views and whose edges represent interpolation trajectories, defining which views to synthesize, which view pairs to interpolate, and in which order generation should proceed. Beyond automatic view scheduling, SceneFrom3D further improves controllability through object-level conditioning, assigning each object an identity image for appearance guidance and a geometry-adherence parameter for region-wise control over the input geometry. Experiments demonstrate that SceneFrom3D achieves state-of-the-art geometry-conditioned outdoor 3D scene generation, producing high-quality scenes with controllable object appearance and geometry adherence.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2607.04540.png","numComments":2,"upvoted":false,"organization":{"_id":"6a1d7b2c2c850ce78fcd590a","name":"postech-cglab","fullname":"POSTECH Computer Graphics Lab","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/65ad0940d63812c33e033124/v90h7lJrY0VAWqHLkMcxM.png"},"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"SceneFrom3D: Geometry-Conditioned Outdoor 3D Scene Generation via View Scheduling with Object-Level Control"}],"highlightedSummary":[{"type":"text","text":"Geometry-conditioned 3D scene generation enables the creation of 3D environments from user-provided geometry, offering direct control over scene structure and object layout. To generate such 3D scenes, current methods commonly adopt a three-stage design that first defines a view schedule, then synthesizes multi-view observations along the scheduled views, and finally reconstructs a 3D representation from the generated images. However, defining the view schedule becomes a major bottleneck for outdoor scenes, where large, unstructured, and unbounded geometry makes it difficult to obtain views that provide sufficient coverage while supporting stable generation. To address this bottleneck, we present SceneFrom3D, a framework that automatically schedules views from outdoor input geometries. "},{"type":"highlight","text":"SceneFrom3D constructs a directed generation graph whose nodes represent anchor views and whose edges represent interpolation trajectories, defining which views to synthesize, which view pairs to interpolate, and in which order generation should proceed."},{"type":"text","text":" Beyond automatic view scheduling, SceneFrom3D further improves controllability through object-level conditioning, assigning each object an identity image for appearance guidance and a geometry-adherence parameter for region-wise control over the input geometry. Experiments demonstrate that SceneFrom3D achieves state-of-the-art geometry-conditioned outdoor 3D scene generation, producing high-quality scenes with controllable object appearance and geometry adherence."}]},{"paper":{"id":"2604.06113","authors":[{"_id":"6a07eff375184a0d71e021ba","name":"Hiba Dahmani","hidden":false},{"_id":"6a07eff375184a0d71e021bb","name":"Nathan Piasco","hidden":false},{"_id":"6a07eff375184a0d71e021bc","name":"Moussab Bennehar","hidden":false},{"_id":"6a07eff375184a0d71e021bd","name":"Luis Roldão","hidden":false},{"_id":"6a07eff375184a0d71e021be","name":"Dzmitry Tsishkou","hidden":false},{"_id":"6a07eff375184a0d71e021bf","name":"Laurent Caraffa","hidden":false},{"_id":"6a07eff375184a0d71e021c0","name":"Jean-Philippe Tarel","hidden":false},{"_id":"6a07eff375184a0d71e021c1","name":"Roland Brémond","hidden":false}],"publishedAt":"2026-04-07T00:00:00.000Z","title":"SEM-ROVER: Semantic Voxel-Guided Diffusion for Large-Scale Driving Scene Generation","summary":"Scalable generation of outdoor driving scenes requires 3D representations that remain consistent across multiple viewpoints and scale to large areas. Existing solutions either rely on image or video generative models distilled to 3D space, harming the geometric coherence and restricting the rendering to training views, or are limited to small-scale 3D scene or object-centric generation. In this work, we propose a 3D generative framework based on Σ-Voxfield grid, a discrete representation where each occupied voxel stores a fixed number of colorized surface samples. To generate this representation, we train a semantic-conditioned diffusion model that operates on local voxel neighborhoods and uses 3D positional encodings to capture spatial structure. We scale to large scenes via progressive spatial outpainting over overlapping regions. Finally, we render the generated Σ-Voxfield grid with a deferred rendering module to obtain photorealistic images, enabling large-scale multiview-consistent 3D scene generation without per-scene optimization. Extensive experiments show that our approach can generate diverse large-scale urban outdoor scenes, renderable into photorealistic images with various sensor configurations and camera trajectories while maintaining moderate computation cost compared to existing approaches.","upvotes":0,"discussionId":"6a07eff475184a0d71e021c2","ai_summary":"A 3D generative framework using Σ-Voxfield grid and semantic-conditioned diffusion models enables large-scale outdoor scene generation with multiview consistency and photorealistic rendering.","ai_keywords":["Σ-Voxfield grid","diffusion model","3D positional encodings","progressive spatial outpainting","deferred rendering","semantic-conditioned diffusion model","multiview-consistent 3D scene generation","photorealistic rendering"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2026-04-06T20:00:00.000Z","title":"SEM-ROVER: Semantic Voxel-Guided Diffusion for Large-Scale Driving Scene Generation","summary":"Scalable generation of outdoor driving scenes requires 3D representations that remain consistent across multiple viewpoints and scale to large areas. Existing solutions either rely on image or video generative models distilled to 3D space, harming the geometric coherence and restricting the rendering to training views, or are limited to small-scale 3D scene or object-centric generation. In this work, we propose a 3D generative framework based on Σ-Voxfield grid, a discrete representation where each occupied voxel stores a fixed number of colorized surface samples. To generate this representation, we train a semantic-conditioned diffusion model that operates on local voxel neighborhoods and uses 3D positional encodings to capture spatial structure. We scale to large scenes via progressive spatial outpainting over overlapping regions. Finally, we render the generated Σ-Voxfield grid with a deferred rendering module to obtain photorealistic images, enabling large-scale multiview-consistent 3D scene generation without per-scene optimization. Extensive experiments show that our approach can generate diverse large-scale urban outdoor scenes, renderable into photorealistic images with various sensor configurations and camera trajectories while maintaining moderate computation cost compared to existing approaches.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2604.06113.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"SEM-ROVER: Semantic Voxel-Guided Diffusion for Large-Scale Driving Scene Generation"}],"highlightedSummary":[{"type":"text","text":"Scalable generation of outdoor driving scenes requires 3D representations that remain consistent across multiple viewpoints and scale to large areas. "},{"type":"highlight","text":"Existing solutions either rely on image or video generative models distilled to 3D space, harming the geometric coherence and restricting the rendering to training views, or are limited to small-scale 3D scene or object-centric generation."},{"type":"text","text":" In this work, we propose a 3D generative framework based on Σ-Voxfield grid, a discrete representation where each occupied voxel stores a fixed number of colorized surface samples. To generate this representation, we train a semantic-conditioned diffusion model that operates on local voxel neighborhoods and uses 3D positional encodings to capture spatial structure. We scale to large scenes via progressive spatial outpainting over overlapping regions. Finally, we render the generated Σ-Voxfield grid with a deferred rendering module to obtain photorealistic images, enabling large-scale multiview-consistent 3D scene generation without per-scene optimization. Extensive experiments show that our approach can generate diverse large-scale urban outdoor scenes, renderable into photorealistic images with various sensor configurations and camera trajectories while maintaining moderate computation cost compared to existing approaches."}]},{"paper":{"id":"2408.11413","authors":[{"_id":"69f7dacd3165501d1edefba5","name":"Guo Pu","hidden":false},{"_id":"69f7dacd3165501d1edefba6","name":"Yiming Zhao","hidden":false},{"_id":"69f7dacd3165501d1edefba7","name":"Zhouhui Lian","hidden":false}],"publishedAt":"2024-08-27T00:00:00.000Z","title":"Pano2Room: Novel View Synthesis from a Single Indoor Panorama","summary":"Recent single-view 3D generative methods have made significant advancements by leveraging knowledge distilled from extensive 3D object datasets. However, challenges persist in the synthesis of 3D scenes from a single view, primarily due to the complexity of real-world environments and the limited availability of high-quality prior resources. In this paper, we introduce a novel approach called Pano2Room, designed to automatically reconstruct high-quality 3D indoor scenes from a single panoramic image. These panoramic images can be easily generated using a panoramic RGBD inpainter from captures at a single location with any camera. The key idea is to initially construct a preliminary mesh from the input panorama, and iteratively refine this mesh using a panoramic RGBD inpainter while collecting photo-realistic 3D-consistent pseudo novel views. Finally, the refined mesh is converted into a 3D Gaussian Splatting field and trained with the collected pseudo novel views. This pipeline enables the reconstruction of real-world 3D scenes, even in the presence of large occlusions, and facilitates the synthesis of photo-realistic novel views with detailed geometry. Extensive qualitative and quantitative experiments have been conducted to validate the superiority of our method in single-panorama indoor novel synthesis compared to the state-of-the-art. Our code and data are available at https://github.com/TrickyGo/Pano2Room.","upvotes":0,"discussionId":"69f7dace3165501d1edefba8","ai_summary":"A novel method called Pano2Room is presented for reconstructing high-quality 3D indoor scenes from single panoramic images through iterative mesh refinement and 3D Gaussian Splatting field training.","ai_keywords":["panoramic RGBD inpainter","3D Gaussian Splatting","mesh refinement","photo-realistic novel views","3D scene reconstruction","single-view 3D generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-08-26T20:00:00.000Z","title":"Pano2Room: Novel View Synthesis from a Single Indoor Panorama","summary":"Recent single-view 3D generative methods have made significant advancements by leveraging knowledge distilled from extensive 3D object datasets. However, challenges persist in the synthesis of 3D scenes from a single view, primarily due to the complexity of real-world environments and the limited availability of high-quality prior resources. In this paper, we introduce a novel approach called Pano2Room, designed to automatically reconstruct high-quality 3D indoor scenes from a single panoramic image. These panoramic images can be easily generated using a panoramic RGBD inpainter from captures at a single location with any camera. The key idea is to initially construct a preliminary mesh from the input panorama, and iteratively refine this mesh using a panoramic RGBD inpainter while collecting photo-realistic 3D-consistent pseudo novel views. Finally, the refined mesh is converted into a 3D Gaussian Splatting field and trained with the collected pseudo novel views. This pipeline enables the reconstruction of real-world 3D scenes, even in the presence of large occlusions, and facilitates the synthesis of photo-realistic novel views with detailed geometry. Extensive qualitative and quantitative experiments have been conducted to validate the superiority of our method in single-panorama indoor novel synthesis compared to the state-of-the-art. Our code and data are available at https://github.com/TrickyGo/Pano2Room.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2408.11413.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Pano2Room: Novel View Synthesis from a Single Indoor Panorama"}],"highlightedSummary":[{"type":"text","text":"Recent single-view 3D generative methods have made significant advancements by leveraging knowledge distilled from extensive 3D object datasets. "},{"type":"highlight","text":"However, challenges persist in the synthesis of 3D scenes from a single view, primarily due to the complexity of real-world environments and the limited availability of high-quality prior resources."},{"type":"text","text":" In this paper, we introduce a novel approach called Pano2Room, designed to automatically reconstruct high-quality 3D indoor scenes from a single panoramic image. These panoramic images can be easily generated using a panoramic RGBD inpainter from captures at a single location with any camera. The key idea is to initially construct a preliminary mesh from the input panorama, and iteratively refine this mesh using a panoramic RGBD inpainter while collecting photo-realistic 3D-consistent pseudo novel views. Finally, the refined mesh is converted into a 3D Gaussian Splatting field and trained with the collected pseudo novel views. This pipeline enables the reconstruction of real-world 3D scenes, even in the presence of large occlusions, and facilitates the synthesis of photo-realistic novel views with detailed geometry. Extensive qualitative and quantitative experiments have been conducted to validate the superiority of our method in single-panorama indoor novel synthesis compared to the state-of-the-art. Our code and data are available at https://github.com/TrickyGo/Pano2Room."}]},{"paper":{"id":"2411.07765","authors":[{"_id":"673491e3b6daf0fbdbad4bfa","name":"Noam Elata","hidden":false},{"_id":"673491e3b6daf0fbdbad4bfb","name":"Bahjat Kawar","hidden":false},{"_id":"673491e3b6daf0fbdbad4bfc","user":{"_id":"686a8d75995e3cddf7f55a03","avatarUrl":"/avatars/3b32c1ff32543ff977dfb34ee7f2dad0.svg","isPro":false,"fullname":"Yaron Ostrovsky-Berman","user":"yostrovskyberman","type":"user","name":"yostrovskyberman"},"name":"Yaron Ostrovsky-Berman","status":"extracted_pending","statusLastChangedAt":"2025-07-06T14:51:47.810Z","hidden":false},{"_id":"673491e3b6daf0fbdbad4bfd","name":"Miriam Farber","hidden":false},{"_id":"673491e3b6daf0fbdbad4bfe","name":"Ron Sokolovsky","hidden":false}],"publishedAt":"2024-11-12T12:58:33.000Z","title":"Novel View Synthesis with Pixel-Space Diffusion Models","summary":"Synthesizing a novel view from a single input image is a challenging task.\nTraditionally, this task was approached by estimating scene depth, warping, and\ninpainting, with machine learning models enabling parts of the pipeline. More\nrecently, generative models are being increasingly employed in novel view\nsynthesis (NVS), often encompassing the entire end-to-end system. In this work,\nwe adapt a modern diffusion model architecture for end-to-end NVS in the pixel\nspace, substantially outperforming previous state-of-the-art (SOTA) techniques.\nWe explore different ways to encode geometric information into the network. Our\nexperiments show that while these methods may enhance performance, their impact\nis minor compared to utilizing improved generative models. Moreover, we\nintroduce a novel NVS training scheme that utilizes single-view datasets,\ncapitalizing on their relative abundance compared to their multi-view\ncounterparts. This leads to improved generalization capabilities to scenes with\nout-of-domain content.","upvotes":0,"discussionId":"673491e4b6daf0fbdbad4c7a","ai_summary":"Adapting a diffusion model architecture for end-to-end novel view synthesis achieves better performance than traditional methods, with novel training schemes using single-view datasets enhancing generalization.","ai_keywords":["diffusion model architecture","novel view synthesis","pixel space","state-of-the-art","generative models","geometric information","single-view datasets","multi-view datasets"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-11-12T07:58:33.000Z","title":"Novel View Synthesis with Pixel-Space Diffusion Models","summary":"Synthesizing a novel view from a single input image is a challenging task.\nTraditionally, this task was approached by estimating scene depth, warping, and\ninpainting, with machine learning models enabling parts of the pipeline. More\nrecently, generative models are being increasingly employed in novel view\nsynthesis (NVS), often encompassing the entire end-to-end system. In this work,\nwe adapt a modern diffusion model architecture for end-to-end NVS in the pixel\nspace, substantially outperforming previous state-of-the-art (SOTA) techniques.\nWe explore different ways to encode geometric information into the network. Our\nexperiments show that while these methods may enhance performance, their impact\nis minor compared to utilizing improved generative models. Moreover, we\nintroduce a novel NVS training scheme that utilizes single-view datasets,\ncapitalizing on their relative abundance compared to their multi-view\ncounterparts. This leads to improved generalization capabilities to scenes with\nout-of-domain content.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2411.07765.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Novel View Synthesis with Pixel-Space Diffusion Models"}],"highlightedSummary":[{"type":"text","text":"Synthesizing a novel view from a single input image is a challenging task.\n"},{"type":"highlight","text":"Traditionally, this task was approached by estimating scene depth, warping, and\ninpainting, with machine learning models enabling parts of the pipeline."},{"type":"text","text":" More\nrecently, generative models are being increasingly employed in novel view\nsynthesis (NVS), often encompassing the entire end-to-end system. In this work,\nwe adapt a modern diffusion model architecture for end-to-end NVS in the pixel\nspace, substantially outperforming previous state-of-the-art (SOTA) techniques.\nWe explore different ways to encode geometric information into the network. Our\nexperiments show that while these methods may enhance performance, their impact\nis minor compared to utilizing improved generative models. Moreover, we\nintroduce a novel NVS training scheme that utilizes single-view datasets,\ncapitalizing on their relative abundance compared to their multi-view\ncounterparts. This leads to improved generalization capabilities to scenes with\nout-of-domain content."}]},{"paper":{"id":"2609.05594","authors":[{"_id":"6aa0d046d0174964227becd6","user":{"_id":"691fe1f6ddc7986706bf86cf","avatarUrl":"/avatars/4f24fd572fa4a07d0d6cfcba8248e6d8.svg","isPro":false,"fullname":"Xingjian Ran","user":"xjran","type":"user","name":"xjran"},"name":"Xingjian Ran","status":"claimed_verified","statusLastChangedAt":"2026-09-09T09:22:49.925Z","hidden":false},{"_id":"6aa0d046d0174964227becd7","name":"Xiaoye Mo","hidden":false},{"_id":"6aa0d046d0174964227becd8","name":"Sihao Liu","hidden":false},{"_id":"6aa0d046d0174964227becd9","name":"Jianyu Zhang","hidden":false},{"_id":"6aa0d046d0174964227becda","user":{"_id":"652e39fea3da41257d6ec253","avatarUrl":"/avatars/fc00bd8bf3704afb269b45b5bf7fd1b9.svg","isPro":false,"fullname":"Li Luo","user":"ReyLock","type":"user","name":"ReyLock"},"name":"Li Luo","status":"claimed_verified","statusLastChangedAt":"2026-09-09T08:45:04.601Z","hidden":false},{"_id":"6aa0d046d0174964227becdb","name":"Bo Dai","hidden":false}],"publishedAt":"2026-09-04T00:00:00.000Z","title":"SceneMosaic: Efficient and Diverse Simulation-Ready Scene Generation via Hybrid Agentic Layout Evolution","summary":"Diverse and simulation-ready indoor scenes are essential for interactive entertainment and embodied AI, yet their scalable generation remains challenging. Recent agentic text-to-3D scene pipelines that rely on vision-language models (VLMs) can generate scenes of high fidelity but require costly iterative object placement and refinement. Another mainstream paradigm, parametric image-to-3D scene models, produces scenes efficiently from strong priors learned from 2D images but often leads to imprecise and physically invalid scenes. More importantly, both paradigms struggle to output diverse scenes for a single input, making it hard for them to reflect the dynamically changing nature of real scenes. In this paper we propose SceneMosaic, a framework that combines the merits of both paradigms. It obtains the initial candidate from the learned image-based prior, and subsequently evolves the result through VLM agents, ensuring both efficiency and physical validity. Within the evolution process, SceneMosaic exploits the locality of natural scenes and decomposes a scene into independent local units, allowing separate evolution within each unit before composing the global scene via Cartesian product. On SceneEval-100, SceneMosaic matches the strongest agentic baseline in semantic layout quality with a 24x speedup, substantially reduces physical violations, and receives the highest human ratings. Our code is publicly available at https://github.com/rxjfighting/SceneMosaic.","upvotes":35,"discussionId":"6aa0d046d0174964227becdc","projectPage":"https://rxjfighting.github.io/SceneMosaic/","githubRepo":"https://github.com/rxjfighting/SceneMosaic","githubRepoAddedBy":"user","ai_summary":"SceneMosaic combines learned image priors with vision-language agents to efficiently generate diverse, physically valid indoor scenes by evolving local units and composing them globally.","ai_keywords":["vision-language models","parametric image-to-3D scene models","SceneMosaic","Cartesian product","SceneEval-100","semantic layout","physical violations"],"ai_summary_model":"thinkingmachines/Inkling-Small","githubStars":45,"organization":{"_id":"67ea9ecfc234715db8dbf339","name":"hkuhk","fullname":"The University of Hong Kong","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/67ea9e8d2d95c10a0da11b0c/FNnR4M7YqKRuG43N5771B.png"}},"publishedAt":"2026-09-03T20:00:00.000Z","title":"SceneMosaic: Efficient and Diverse Simulation-Ready Scene Generation via Hybrid Agentic Layout Evolution","summary":"Diverse and simulation-ready indoor scenes are essential for interactive entertainment and embodied AI, yet their scalable generation remains challenging. Recent agentic text-to-3D scene pipelines that rely on vision-language models (VLMs) can generate scenes of high fidelity but require costly iterative object placement and refinement. Another mainstream paradigm, parametric image-to-3D scene models, produces scenes efficiently from strong priors learned from 2D images but often leads to imprecise and physically invalid scenes. More importantly, both paradigms struggle to output diverse scenes for a single input, making it hard for them to reflect the dynamically changing nature of real scenes. In this paper we propose SceneMosaic, a framework that combines the merits of both paradigms. It obtains the initial candidate from the learned image-based prior, and subsequently evolves the result through VLM agents, ensuring both efficiency and physical validity. Within the evolution process, SceneMosaic exploits the locality of natural scenes and decomposes a scene into independent local units, allowing separate evolution within each unit before composing the global scene via Cartesian product. On SceneEval-100, SceneMosaic matches the strongest agentic baseline in semantic layout quality with a 24x speedup, substantially reduces physical violations, and receives the highest human ratings. Our code is publicly available at https://github.com/rxjfighting/SceneMosaic.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2609.05594.png","numComments":3,"upvoted":false,"organization":{"_id":"67ea9ecfc234715db8dbf339","name":"hkuhk","fullname":"The University of Hong Kong","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/67ea9e8d2d95c10a0da11b0c/FNnR4M7YqKRuG43N5771B.png"},"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"SceneMosaic: Efficient and Diverse Simulation-Ready Scene Generation via Hybrid Agentic Layout Evolution"}],"highlightedSummary":[{"type":"text","text":"Diverse and simulation-ready indoor scenes are essential for interactive entertainment and embodied AI, yet their scalable generation remains challenging. Recent agentic text-to-3D scene pipelines that rely on vision-language models (VLMs) can generate scenes of high fidelity but require costly iterative object placement and refinement. Another mainstream paradigm, parametric image-to-3D scene models, produces scenes efficiently from strong priors learned from 2D images but often leads to imprecise and physically invalid scenes. More importantly, both paradigms struggle to output diverse scenes for a single input, making it hard for them to reflect the dynamically changing nature of real scenes. In this paper we propose SceneMosaic, a framework that combines the merits of both paradigms. It obtains the initial candidate from the learned image-based prior, and subsequently evolves the result through VLM agents, ensuring both efficiency and physical validity. Within the evolution process, SceneMosaic exploits the locality of natural scenes and decomposes a scene into independent local units, allowing separate evolution within each unit before composing the global scene via Cartesian product. On SceneEval-100, SceneMosaic matches the strongest agentic baseline in semantic layout quality with a 24x speedup, substantially reduces physical violations, and receives the highest human ratings. Our code is publicly available at https://github.com/rxjfighting/SceneMosaic."}]},{"paper":{"id":"2403.09675","authors":[{"_id":"6672a5354939264251b6a6df","user":{"_id":"677ebf1cae26cefc2dc1120e","avatarUrl":"/avatars/78b44716d341c90a781510a8ba469615.svg","isPro":false,"fullname":"Rio Aguina-Kang","user":"raksak","type":"user","name":"raksak"},"name":"Rio Aguina-Kang","status":"extracted_pending","statusLastChangedAt":"2025-08-18T15:53:30.836Z","hidden":false},{"_id":"6672a5354939264251b6a6e0","name":"Maxim Gumin","hidden":false},{"_id":"6672a5354939264251b6a6e1","name":"Do Heon Han","hidden":false},{"_id":"6672a5354939264251b6a6e2","name":"Stewart Morris","hidden":false},{"_id":"6672a5354939264251b6a6e3","name":"Seung Jean Yoo","hidden":false},{"_id":"6672a5354939264251b6a6e4","name":"Aditya Ganeshan","hidden":false},{"_id":"6672a5354939264251b6a6e5","user":{"_id":"67cb602e5432f525e856506c","avatarUrl":"/avatars/939078fe3cc9c6a8ca64b8d60bc377f7.svg","isPro":false,"fullname":"Kenny Jones","user":"rkjones4","type":"user","name":"rkjones4"},"name":"R. Kenny Jones","status":"extracted_pending","statusLastChangedAt":"2025-03-07T21:08:13.722Z","hidden":false},{"_id":"6672a5354939264251b6a6e6","name":"Qiuhong Anna Wei","hidden":false},{"_id":"6672a5354939264251b6a6e7","name":"Kailiang Fu","hidden":false},{"_id":"6672a5354939264251b6a6e8","name":"Daniel Ritchie","hidden":false}],"publishedAt":"2024-02-05T01:59:31.000Z","title":"Open-Universe Indoor Scene Generation using LLM Program Synthesis and\n  Uncurated Object Databases","summary":"We present a system for generating indoor scenes in response to text prompts.\nThe prompts are not limited to a fixed vocabulary of scene descriptions, and\nthe objects in generated scenes are not restricted to a fixed set of object\ncategories -- we call this setting indoor scene generation. Unlike most prior\nwork on indoor scene generation, our system does not require a large training\ndataset of existing 3D scenes. Instead, it leverages the world knowledge\nencoded in pre-trained large language models (LLMs) to synthesize programs in a\ndomain-specific layout language that describe objects and spatial relations\nbetween them. Executing such a program produces a specification of a constraint\nsatisfaction problem, which the system solves using a gradient-based\noptimization scheme to produce object positions and orientations. To produce\nobject geometry, the system retrieves 3D meshes from a database. Unlike prior\nwork which uses databases of category-annotated, mutually-aligned meshes, we\ndevelop a pipeline using vision-language models (VLMs) to retrieve meshes from\nmassive databases of un-annotated, inconsistently-aligned meshes. Experimental\nevaluations show that our system outperforms generative models trained on 3D\ndata for traditional, closed-universe scene generation tasks; it also\noutperforms a recent LLM-based layout generation method on open-universe scene\ngeneration.","upvotes":0,"discussionId":"6672a53a4939264251b6a895","ai_summary":"A system generates indoor scenes from text prompts using pre-trained language models to synthesize programs in a layout language and gradient-based optimization, outperforming models trained on 3D data and layout generation methods on traditional and open-universe tasks.","ai_keywords":["pre-trained large language models (LLMs)","domain-specific layout language","constraint satisfaction problem","gradient-based optimization scheme","vision-language models (VLMs)","3D meshes","closed-universe scene generation","open-universe scene generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-02-04T20:59:31.000Z","title":"Open-Universe Indoor Scene Generation using LLM Program Synthesis and\n  Uncurated Object Databases","summary":"We present a system for generating indoor scenes in response to text prompts.\nThe prompts are not limited to a fixed vocabulary of scene descriptions, and\nthe objects in generated scenes are not restricted to a fixed set of object\ncategories -- we call this setting indoor scene generation. Unlike most prior\nwork on indoor scene generation, our system does not require a large training\ndataset of existing 3D scenes. Instead, it leverages the world knowledge\nencoded in pre-trained large language models (LLMs) to synthesize programs in a\ndomain-specific layout language that describe objects and spatial relations\nbetween them. Executing such a program produces a specification of a constraint\nsatisfaction problem, which the system solves using a gradient-based\noptimization scheme to produce object positions and orientations. To produce\nobject geometry, the system retrieves 3D meshes from a database. Unlike prior\nwork which uses databases of category-annotated, mutually-aligned meshes, we\ndevelop a pipeline using vision-language models (VLMs) to retrieve meshes from\nmassive databases of un-annotated, inconsistently-aligned meshes. Experimental\nevaluations show that our system outperforms generative models trained on 3D\ndata for traditional, closed-universe scene generation tasks; it also\noutperforms a recent LLM-based layout generation method on open-universe scene\ngeneration.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2403.09675.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Open-Universe Indoor Scene Generation using LLM Program Synthesis and\n  Uncurated Object Databases"}],"highlightedSummary":[{"type":"text","text":"We present a system for generating indoor scenes in response to text prompts.\nThe prompts are not limited to a fixed vocabulary of scene descriptions, and\nthe objects in generated scenes are not restricted to a fixed set of object\ncategories -- we call this setting indoor scene generation. Unlike most prior\nwork on indoor scene generation, our system does not require a large training\ndataset of existing 3D scenes. Instead, it leverages the world knowledge\nencoded in pre-trained large language models (LLMs) to synthesize programs in a\ndomain-specific layout language that describe objects and spatial relations\nbetween them. Executing such a program produces a specification of a constraint\nsatisfaction problem, which the system solves using a gradient-based\noptimization scheme to produce object positions and orientations. To produce\nobject geometry, the system retrieves 3D meshes from a database. Unlike prior\nwork which uses databases of category-annotated, mutually-aligned meshes, we\ndevelop a pipeline using vision-language models (VLMs) to retrieve meshes from\nmassive databases of un-annotated, inconsistently-aligned meshes. "},{"type":"highlight","text":"Experimental\nevaluations show that our system outperforms generative models trained on 3D\ndata for traditional, closed-universe scene generation tasks; it also\noutperforms a recent LLM-based layout generation method on open-universe scene\ngeneration."}]},{"paper":{"id":"2607.16805","authors":[{"_id":"6a97e642d4c640d9b759ba19","name":"Yuqi Zhang","hidden":false},{"_id":"6a97e642d4c640d9b759ba1a","name":"Yadan Luo","hidden":false},{"_id":"6a97e642d4c640d9b759ba1b","name":"Xiangyu Sun","hidden":false},{"_id":"6a97e642d4c640d9b759ba1c","name":"Fengyi Zhang","hidden":false},{"_id":"6a97e642d4c640d9b759ba1d","name":"Zi Huang","hidden":false},{"_id":"6a97e642d4c640d9b759ba1e","name":"Xin Tan","hidden":false}],"publishedAt":"2026-07-18T00:00:00.000Z","title":"Scene-SAM3D: Multi-View Scene Asset Generation Without Fine-Tuning","summary":"High-quality 3D scene assets are critical for embodied applications such as robotic manipulation, navigation, and simulation. Despite their strong object priors, recent single-image 3D generation models such as SAM3D remain insufficient for real-world scenes, where severe occlusions, redundant observations, and cross-view inconsistencies make reliable scene generation challenging. We introduce Scene-SAM3D, a training-free framework that extends SAM3D from single-view object generation to calibrated multi-view scene asset generation. Scene-SAM3D selects a compact set of complementary views, reducing observation redundancy while providing additional evidence for regions occluded in individual views. Based on the selected views, it performs step-efficient latent velocity fusion to integrate multi-view evidence and suppress cross-view conflicts in canonical space. Finally, a lightweight rigid-object Gaussian optimization refines the scene layout within 200 iterations while preserving the generated object geometry. Experiments on Replica and ScanNet++ demonstrate consistent improvements at both instance and scene levels, with our method reducing scene-level CD by 43.8% on Replica and 30.9% on ScanNet++, while cutting flow-model sampling FLOPs and wall-time latency by nearly 20% under the same multi-view setting. Code will be released at https://github.com/xibi777/Scene-SAM3D.","upvotes":0,"discussionId":"6a97e642d4c640d9b759ba1f","ai_summary":"Scene-SAM3D extends single-view 3D generation to multi-view calibrated scenes via complementary view selection, latent velocity fusion, and rapid Gaussian optimization, improving reconstruction accuracy and efficiency.","ai_keywords":["single-image 3D generation","SAM3D","multi-view scene generation","latent velocity fusion","Gaussian optimization","cross-view consistency","scene-level reconstruction"],"ai_summary_model":"thinkingmachines/Inkling-Small"},"publishedAt":"2026-07-17T20:00:00.000Z","title":"Scene-SAM3D: Multi-View Scene Asset Generation Without Fine-Tuning","summary":"High-quality 3D scene assets are critical for embodied applications such as robotic manipulation, navigation, and simulation. Despite their strong object priors, recent single-image 3D generation models such as SAM3D remain insufficient for real-world scenes, where severe occlusions, redundant observations, and cross-view inconsistencies make reliable scene generation challenging. We introduce Scene-SAM3D, a training-free framework that extends SAM3D from single-view object generation to calibrated multi-view scene asset generation. Scene-SAM3D selects a compact set of complementary views, reducing observation redundancy while providing additional evidence for regions occluded in individual views. Based on the selected views, it performs step-efficient latent velocity fusion to integrate multi-view evidence and suppress cross-view conflicts in canonical space. Finally, a lightweight rigid-object Gaussian optimization refines the scene layout within 200 iterations while preserving the generated object geometry. Experiments on Replica and ScanNet++ demonstrate consistent improvements at both instance and scene levels, with our method reducing scene-level CD by 43.8% on Replica and 30.9% on ScanNet++, while cutting flow-model sampling FLOPs and wall-time latency by nearly 20% under the same multi-view setting. Code will be released at https://github.com/xibi777/Scene-SAM3D.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2607.16805.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Scene-SAM3D: Multi-View Scene Asset Generation Without Fine-Tuning"}],"highlightedSummary":[{"type":"text","text":"High-quality 3D scene assets are critical for embodied applications such as robotic manipulation, navigation, and simulation. "},{"type":"highlight","text":"Despite their strong object priors, recent single-image 3D generation models such as SAM3D remain insufficient for real-world scenes, where severe occlusions, redundant observations, and cross-view inconsistencies make reliable scene generation challenging."},{"type":"text","text":" We introduce Scene-SAM3D, a training-free framework that extends SAM3D from single-view object generation to calibrated multi-view scene asset generation. Scene-SAM3D selects a compact set of complementary views, reducing observation redundancy while providing additional evidence for regions occluded in individual views. Based on the selected views, it performs step-efficient latent velocity fusion to integrate multi-view evidence and suppress cross-view conflicts in canonical space. Finally, a lightweight rigid-object Gaussian optimization refines the scene layout within 200 iterations while preserving the generated object geometry. Experiments on Replica and ScanNet++ demonstrate consistent improvements at both instance and scene levels, with our method reducing scene-level CD by 43.8% on Replica and 30.9% on ScanNet++, while cutting flow-model sampling FLOPs and wall-time latency by nearly 20% under the same multi-view setting. Code will be released at https://github.com/xibi777/Scene-SAM3D."}]},{"paper":{"id":"2506.09980","authors":[{"_id":"684b7c563b733ba333686f7e","name":"Jiaxiang Tang","hidden":false},{"_id":"684b7c563b733ba333686f7f","name":"Ruijie Lu","hidden":false},{"_id":"684b7c563b733ba333686f80","name":"Zhaoshuo Li","hidden":false},{"_id":"684b7c563b733ba333686f81","name":"Zekun Hao","hidden":false},{"_id":"684b7c563b733ba333686f82","name":"Xuan Li","hidden":false},{"_id":"684b7c563b733ba333686f83","name":"Fangyin Wei","hidden":false},{"_id":"684b7c563b733ba333686f84","name":"Shuran Song","hidden":false},{"_id":"684b7c563b733ba333686f85","name":"Gang Zeng","hidden":false},{"_id":"684b7c563b733ba333686f86","name":"Ming-Yu Liu","hidden":false},{"_id":"684b7c563b733ba333686f87","name":"Tsung-Yi Lin","hidden":false}],"publishedAt":"2025-06-11T17:55:03.000Z","title":"Efficient Part-level 3D Object Generation via Dual Volume Packing","summary":"Recent progress in 3D object generation has greatly improved both the quality\nand efficiency. However, most existing methods generate a single mesh with all\nparts fused together, which limits the ability to edit or manipulate individual\nparts. A key challenge is that different objects may have a varying number of\nparts. To address this, we propose a new end-to-end framework for part-level 3D\nobject generation. Given a single input image, our method generates\nhigh-quality 3D objects with an arbitrary number of complete and semantically\nmeaningful parts. We introduce a dual volume packing strategy that organizes\nall parts into two complementary volumes, allowing for the creation of complete\nand interleaved parts that assemble into the final object. Experiments show\nthat our model achieves better quality, diversity, and generalization than\nprevious image-based part-level generation methods.","upvotes":8,"discussionId":"684b7c563b733ba333686f88","projectPage":"https://research.nvidia.com/labs/dir/partpacker/","githubRepo":"https://github.com/NVlabs/PartPacker","githubRepoAddedBy":"user","ai_summary":"A new end-to-end framework generates high-quality 3D objects with part-level detail from a single image using a dual volume packing strategy.","ai_keywords":["3D object generation","part-level generation","dual volume packing strategy","image-based generation","semantic meaning","complete parts","interleaved parts"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":824},"publishedAt":"2025-06-11T13:55:03.000Z","title":"Efficient Part-level 3D Object Generation via Dual Volume Packing","summary":"Recent progress in 3D object generation has greatly improved both the quality\nand efficiency. However, most existing methods generate a single mesh with all\nparts fused together, which limits the ability to edit or manipulate individual\nparts. A key challenge is that different objects may have a varying number of\nparts. To address this, we propose a new end-to-end framework for part-level 3D\nobject generation. Given a single input image, our method generates\nhigh-quality 3D objects with an arbitrary number of complete and semantically\nmeaningful parts. We introduce a dual volume packing strategy that organizes\nall parts into two complementary volumes, allowing for the creation of complete\nand interleaved parts that assemble into the final object. Experiments show\nthat our model achieves better quality, diversity, and generalization than\nprevious image-based part-level generation methods.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2506.09980.png","numComments":2,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Efficient Part-level 3D Object Generation via Dual Volume Packing"}],"highlightedSummary":[{"type":"text","text":"Recent progress in 3D object generation has greatly improved both the quality\nand efficiency. However, most existing methods generate a single mesh with all\nparts fused together, which limits the ability to edit or manipulate individual\nparts. A key challenge is that different objects may have a varying number of\nparts. To address this, we propose a new end-to-end framework for part-level 3D\nobject generation. "},{"type":"highlight","text":"Given a single input image, our method generates\nhigh-quality 3D objects with an arbitrary number of complete and semantically\nmeaningful parts."},{"type":"text","text":" We introduce a dual volume packing strategy that organizes\nall parts into two complementary volumes, allowing for the creation of complete\nand interleaved parts that assemble into the final object. Experiments show\nthat our model achieves better quality, diversity, and generalization than\nprevious image-based part-level generation methods."}]},{"paper":{"id":"2309.07920","authors":[{"_id":"660a17d2c30c49ec53ebe122","name":"Ziang Cao","hidden":false},{"_id":"660a17d2c30c49ec53ebe123","name":"Fangzhou Hong","hidden":false},{"_id":"660a17d2c30c49ec53ebe124","user":{"_id":"6310929fda0aaad2f70397a1","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6310929fda0aaad2f70397a1/YoHeYZezmCOXwL-qsiZOc.png","isPro":false,"fullname":"Tong Wu","user":"wutong16","type":"user","name":"wutong16"},"name":"Tong Wu","status":"claimed_verified","statusLastChangedAt":"2024-12-11T09:24:47.526Z","hidden":false},{"_id":"660a17d2c30c49ec53ebe125","user":{"_id":"64ef30fb0de94d31b7921175","avatarUrl":"/avatars/d4f4c7be460befc1cedec13bbf9db972.svg","isPro":false,"fullname":"pan","user":"pldeqiushui","type":"user","name":"pldeqiushui"},"name":"Liang Pan","status":"claimed_verified","statusLastChangedAt":"2024-11-05T16:12:49.008Z","hidden":false},{"_id":"660a17d2c30c49ec53ebe126","name":"Ziwei Liu","hidden":false}],"publishedAt":"2023-09-14T17:59:53.000Z","title":"Large-Vocabulary 3D Diffusion Model with Transformer","summary":"Creating diverse and high-quality 3D assets with an automatic generative\nmodel is highly desirable. Despite extensive efforts on 3D generation, most\nexisting works focus on the generation of a single category or a few\ncategories. In this paper, we introduce a diffusion-based feed-forward\nframework for synthesizing massive categories of real-world 3D objects with a\nsingle generative model. Notably, there are three major challenges for this\nlarge-vocabulary 3D generation: a) the need for expressive yet efficient 3D\nrepresentation; b) large diversity in geometry and texture across categories;\nc) complexity in the appearances of real-world objects. To this end, we propose\na novel triplane-based 3D-aware Diffusion model with TransFormer, DiffTF, for\nhandling challenges via three aspects. 1) Considering efficiency and\nrobustness, we adopt a revised triplane representation and improve the fitting\nspeed and accuracy. 2) To handle the drastic variations in geometry and\ntexture, we regard the features of all 3D objects as a combination of\ngeneralized 3D knowledge and specialized 3D features. To extract generalized 3D\nknowledge from diverse categories, we propose a novel 3D-aware transformer with\nshared cross-plane attention. It learns the cross-plane relations across\ndifferent planes and aggregates the generalized 3D knowledge with specialized\n3D features. 3) In addition, we devise the 3D-aware encoder/decoder to enhance\nthe generalized 3D knowledge in the encoded triplanes for handling categories\nwith complex appearances. Extensive experiments on ShapeNet and OmniObject3D\n(over 200 diverse real-world categories) convincingly demonstrate that a single\nDiffTF model achieves state-of-the-art large-vocabulary 3D object generation\nperformance with large diversity, rich semantics, and high quality.","upvotes":0,"discussionId":"660a17d5c30c49ec53ebe1c3","ai_summary":"A diffusion-based generative model using a triplane representation and 3D-aware transformer excels in generating a diverse range of 3D objects across many categories with high quality and rich semantics.","ai_keywords":["diffusion-based feed-forward framework","diffusion model","triplane representation","3D-aware transformer","shared cross-plane attention","3D-aware encoder/decoder","large-vocabulary 3D generation","ShapeNet","OmniObject3D","generalized 3D knowledge","specialized 3D features"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2023-09-14T13:59:53.000Z","title":"Large-Vocabulary 3D Diffusion Model with Transformer","summary":"Creating diverse and high-quality 3D assets with an automatic generative\nmodel is highly desirable. Despite extensive efforts on 3D generation, most\nexisting works focus on the generation of a single category or a few\ncategories. In this paper, we introduce a diffusion-based feed-forward\nframework for synthesizing massive categories of real-world 3D objects with a\nsingle generative model. Notably, there are three major challenges for this\nlarge-vocabulary 3D generation: a) the need for expressive yet efficient 3D\nrepresentation; b) large diversity in geometry and texture across categories;\nc) complexity in the appearances of real-world objects. To this end, we propose\na novel triplane-based 3D-aware Diffusion model with TransFormer, DiffTF, for\nhandling challenges via three aspects. 1) Considering efficiency and\nrobustness, we adopt a revised triplane representation and improve the fitting\nspeed and accuracy. 2) To handle the drastic variations in geometry and\ntexture, we regard the features of all 3D objects as a combination of\ngeneralized 3D knowledge and specialized 3D features. To extract generalized 3D\nknowledge from diverse categories, we propose a novel 3D-aware transformer with\nshared cross-plane attention. It learns the cross-plane relations across\ndifferent planes and aggregates the generalized 3D knowledge with specialized\n3D features. 3) In addition, we devise the 3D-aware encoder/decoder to enhance\nthe generalized 3D knowledge in the encoded triplanes for handling categories\nwith complex appearances. Extensive experiments on ShapeNet and OmniObject3D\n(over 200 diverse real-world categories) convincingly demonstrate that a single\nDiffTF model achieves state-of-the-art large-vocabulary 3D object generation\nperformance with large diversity, rich semantics, and high quality.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2309.07920.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Large-Vocabulary 3D Diffusion Model with Transformer"}],"highlightedSummary":[{"type":"text","text":"Creating diverse and high-quality 3D assets with an automatic generative\nmodel is highly desirable. "},{"type":"highlight","text":"Despite extensive efforts on 3D generation, most\nexisting works focus on the generation of a single category or a few\ncategories."},{"type":"text","text":" In this paper, we introduce a diffusion-based feed-forward\nframework for synthesizing massive categories of real-world 3D objects with a\nsingle generative model. Notably, there are three major challenges for this\nlarge-vocabulary 3D generation: a) the need for expressive yet efficient 3D\nrepresentation; b) large diversity in geometry and texture across categories;\nc) complexity in the appearances of real-world objects. To this end, we propose\na novel triplane-based 3D-aware Diffusion model with TransFormer, DiffTF, for\nhandling challenges via three aspects. 1) Considering efficiency and\nrobustness, we adopt a revised triplane representation and improve the fitting\nspeed and accuracy. 2) To handle the drastic variations in geometry and\ntexture, we regard the features of all 3D objects as a combination of\ngeneralized 3D knowledge and specialized 3D features. To extract generalized 3D\nknowledge from diverse categories, we propose a novel 3D-aware transformer with\nshared cross-plane attention. It learns the cross-plane relations across\ndifferent planes and aggregates the generalized 3D knowledge with specialized\n3D features. 3) In addition, we devise the 3D-aware encoder/decoder to enhance\nthe generalized 3D knowledge in the encoded triplanes for handling categories\nwith complex appearances. Extensive experiments on ShapeNet and OmniObject3D\n(over 200 diverse real-world categories) convincingly demonstrate that a single\nDiffTF model achieves state-of-the-art large-vocabulary 3D object generation\nperformance with large diversity, rich semantics, and high quality."}]},{"paper":{"id":"2509.21278","authors":[{"_id":"68d5f6788ccd91bdd39ffdb3","user":{"_id":"631c4a23aa346997917bcb89","avatarUrl":"/avatars/4a562ba78e6be289049027c27798cc6e.svg","isPro":false,"fullname":"Shilin Lu","user":"Shilin-LU","type":"user","name":"Shilin-LU"},"name":"Shilin Lu","status":"claimed_verified","statusLastChangedAt":"2025-09-26T12:19:30.705Z","hidden":false},{"_id":"68d5f6788ccd91bdd39ffdb4","name":"Zhuming Lian","hidden":false},{"_id":"68d5f6788ccd91bdd39ffdb5","user":{"_id":"66923522e32997bdf7f37462","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/no-auth/-RE4FqyZnDF9PcpY6ZBGH.png","isPro":false,"fullname":"Zihan Zhou","user":"Edennnnn","type":"user","name":"Edennnnn"},"name":"Zihan Zhou","status":"claimed_verified","statusLastChangedAt":"2025-09-26T12:16:56.963Z","hidden":false},{"_id":"68d5f6788ccd91bdd39ffdb6","name":"Shaocong Zhang","hidden":false},{"_id":"68d5f6788ccd91bdd39ffdb7","name":"Chen Zhao","hidden":false},{"_id":"68d5f6788ccd91bdd39ffdb8","name":"Adams Wai-Kin Kong","hidden":false}],"publishedAt":"2025-09-25T15:01:49.000Z","title":"Does FLUX Already Know How to Perform Physically Plausible Image\n  Composition?","summary":"Image composition aims to seamlessly insert a user-specified object into a\nnew scene, but existing models struggle with complex lighting (e.g., accurate\nshadows, water reflections) and diverse, high-resolution inputs. Modern\ntext-to-image diffusion models (e.g., SD3.5, FLUX) already encode essential\nphysical and resolution priors, yet lack a framework to unleash them without\nresorting to latent inversion, which often locks object poses into contextually\ninappropriate orientations, or brittle attention surgery. We propose SHINE, a\ntraining-free framework for Seamless, High-fidelity Insertion with Neutralized\nErrors. SHINE introduces manifold-steered anchor loss, leveraging pretrained\ncustomization adapters (e.g., IP-Adapter) to guide latents for faithful subject\nrepresentation while preserving background integrity. Degradation-suppression\nguidance and adaptive background blending are proposed to further eliminate\nlow-quality outputs and visible seams. To address the lack of rigorous\nbenchmarks, we introduce ComplexCompo, featuring diverse resolutions and\nchallenging conditions such as low lighting, strong illumination, intricate\nshadows, and reflective surfaces. Experiments on ComplexCompo and\nDreamEditBench show state-of-the-art performance on standard metrics (e.g.,\nDINOv2) and human-aligned scores (e.g., DreamSim, ImageReward, VisionReward).\nCode and benchmark will be publicly available upon publication.","upvotes":16,"discussionId":"68d5f6788ccd91bdd39ffdb9","githubRepo":"https://github.com/ZhumingLian/SHINE","githubRepoAddedBy":"user","ai_summary":"SHINE is a training-free framework that uses manifold-steered anchor loss and pretrained customization adapters to seamlessly insert objects into new scenes with high fidelity, addressing challenges like complex lighting and diverse inputs.","ai_keywords":["text-to-image diffusion models","SD3.5","FLUX","manifold-steered anchor loss","pretrained customization adapters","IP-Adapter","degradation-suppression guidance","adaptive background blending","ComplexCompo","DreamEditBench","DINOv2","DreamSim","ImageReward","VisionReward"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":222,"organization":{"_id":"6508b28cf36bb51c50faad98","name":"NanyangTechnologicalUniversity","fullname":"Nanyang Technological University","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/630ca0817dacb93b33506ce7/ZPD1fvei0bcIGeDXxeSkn.png"}},"publishedAt":"2025-09-25T11:01:49.000Z","title":"Does FLUX Already Know How to Perform Physically Plausible Image\n  Composition?","summary":"Image composition aims to seamlessly insert a user-specified object into a\nnew scene, but existing models struggle with complex lighting (e.g., accurate\nshadows, water reflections) and diverse, high-resolution inputs. Modern\ntext-to-image diffusion models (e.g., SD3.5, FLUX) already encode essential\nphysical and resolution priors, yet lack a framework to unleash them without\nresorting to latent inversion, which often locks object poses into contextually\ninappropriate orientations, or brittle attention surgery. We propose SHINE, a\ntraining-free framework for Seamless, High-fidelity Insertion with Neutralized\nErrors. SHINE introduces manifold-steered anchor loss, leveraging pretrained\ncustomization adapters (e.g., IP-Adapter) to guide latents for faithful subject\nrepresentation while preserving background integrity. Degradation-suppression\nguidance and adaptive background blending are proposed to further eliminate\nlow-quality outputs and visible seams. To address the lack of rigorous\nbenchmarks, we introduce ComplexCompo, featuring diverse resolutions and\nchallenging conditions such as low lighting, strong illumination, intricate\nshadows, and reflective surfaces. Experiments on ComplexCompo and\nDreamEditBench show state-of-the-art performance on standard metrics (e.g.,\nDINOv2) and human-aligned scores (e.g., DreamSim, ImageReward, VisionReward).\nCode and benchmark will be publicly available upon publication.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2509.21278.png","numComments":5,"upvoted":false,"organization":{"_id":"6508b28cf36bb51c50faad98","name":"NanyangTechnologicalUniversity","fullname":"Nanyang Technological University","avatar":"https://cdn-avatars.huggingface.co/v1/production/uploads/630ca0817dacb93b33506ce7/ZPD1fvei0bcIGeDXxeSkn.png"},"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Does FLUX Already Know How to Perform Physically Plausible Image\n  Composition?"}],"highlightedSummary":[{"type":"highlight","text":"Image composition aims to seamlessly insert a user-specified object into a\nnew scene, but existing models struggle with complex lighting (e.g., accurate\nshadows, water reflections) and diverse, high-resolution inputs."},{"type":"text","text":" Modern\ntext-to-image diffusion models (e.g., SD3.5, FLUX) already encode essential\nphysical and resolution priors, yet lack a framework to unleash them without\nresorting to latent inversion, which often locks object poses into contextually\ninappropriate orientations, or brittle attention surgery. We propose SHINE, a\ntraining-free framework for Seamless, High-fidelity Insertion with Neutralized\nErrors. SHINE introduces manifold-steered anchor loss, leveraging pretrained\ncustomization adapters (e.g., IP-Adapter) to guide latents for faithful subject\nrepresentation while preserving background integrity. Degradation-suppression\nguidance and adaptive background blending are proposed to further eliminate\nlow-quality outputs and visible seams. To address the lack of rigorous\nbenchmarks, we introduce ComplexCompo, featuring diverse resolutions and\nchallenging conditions such as low lighting, strong illumination, intricate\nshadows, and reflective surfaces. Experiments on ComplexCompo and\nDreamEditBench show state-of-the-art performance on standard metrics (e.g.,\nDINOv2) and human-aligned scores (e.g., DreamSim, ImageReward, VisionReward).\nCode and benchmark will be publicly available upon publication."}]},{"paper":{"id":"2403.12010","authors":[{"_id":"6611a43be84ec1efeca5eb97","user":{"_id":"64d0d72e15b26cc7f704a60f","avatarUrl":"/avatars/f450013fb2b204f1422e614629914248.svg","isPro":false,"fullname":"Qi Zuo","user":"DyrusQZ","type":"user","name":"DyrusQZ"},"name":"Qi Zuo","status":"claimed_verified","statusLastChangedAt":"2025-03-19T13:16:19.313Z","hidden":false},{"_id":"6611a43be84ec1efeca5eb98","user":{"_id":"6624ede5478dada13051af60","avatarUrl":"/avatars/1b064f7e3c417f1b603909ff3dbf2430.svg","isPro":false,"fullname":"xiaodong gu","user":"guxiaodong1987","type":"user","name":"guxiaodong1987"},"name":"Xiaodong Gu","status":"claimed_verified","statusLastChangedAt":"2026-01-28T11:35:45.238Z","hidden":false},{"_id":"6611a43be84ec1efeca5eb99","name":"Lingteng Qiu","hidden":false},{"_id":"6611a43be84ec1efeca5eb9a","name":"Yuan Dong","hidden":false},{"_id":"6611a43be84ec1efeca5eb9b","name":"Zhengyi Zhao","hidden":false},{"_id":"6611a43be84ec1efeca5eb9c","name":"Weihao Yuan","hidden":false},{"_id":"6611a43be84ec1efeca5eb9d","name":"Rui Peng","hidden":false},{"_id":"6611a43be84ec1efeca5eb9e","name":"Siyu Zhu","hidden":false},{"_id":"6611a43be84ec1efeca5eb9f","name":"Zilong Dong","hidden":false},{"_id":"6611a43be84ec1efeca5eba0","name":"Liefeng Bo","hidden":false},{"_id":"6611a43be84ec1efeca5eba1","user":{"_id":"67f18d051358145ee5bb3a11","avatarUrl":"/avatars/e5a8c12295d7609625c2b891d47c3112.svg","isPro":false,"fullname":"qixing huang","user":"qixinghuang1","type":"user","name":"qixinghuang1"},"name":"Qixing Huang","status":"claimed_verified","statusLastChangedAt":"2025-04-08T07:01:58.359Z","hidden":false}],"publishedAt":"2024-03-18T17:48:15.000Z","title":"VideoMV: Consistent Multi-View Generation Based on Large Video\n  Generative Model","summary":"Generating multi-view images based on text or single-image prompts is a\ncritical capability for the creation of 3D content. Two fundamental questions\non this topic are what data we use for training and how to ensure multi-view\nconsistency. This paper introduces a novel framework that makes fundamental\ncontributions to both questions. Unlike leveraging images from 2D diffusion\nmodels for training, we propose a dense consistent multi-view generation model\nthat is fine-tuned from off-the-shelf video generative models. Images from\nvideo generative models are more suitable for multi-view generation because the\nunderlying network architecture that generates them employs a temporal module\nto enforce frame consistency. Moreover, the video data sets used to train these\nmodels are abundant and diverse, leading to a reduced train-finetuning domain\ngap. To enhance multi-view consistency, we introduce a 3D-Aware Denoising\nSampling, which first employs a feed-forward reconstruction module to get an\nexplicit global 3D model, and then adopts a sampling strategy that effectively\ninvolves images rendered from the global 3D model into the denoising sampling\nloop to improve the multi-view consistency of the final images. As a\nby-product, this module also provides a fast way to create 3D assets\nrepresented by 3D Gaussians within a few seconds. Our approach can generate 24\ndense views and converges much faster in training than state-of-the-art\napproaches (4 GPU hours versus many thousand GPU hours) with comparable visual\nquality and consistency. By further fine-tuning, our approach outperforms\nexisting state-of-the-art methods in both quantitative metrics and visual\neffects. Our project page is aigc3d.github.io/VideoMV.","upvotes":0,"discussionId":"6611a441e84ec1efeca5ed27","ai_summary":"A framework fine-tunes video generative models for multi-view image generation from text or single-image prompts using a 3D-Aware Denoising Sampling method, achieving fast training and high-quality, consistent 3D content generation.","ai_keywords":["video generative models","temporal module","3D-Aware Denoising Sampling","feed-forward reconstruction module","denoising sampling loop","3D Gaussians","multi-view generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-03-18T13:48:15.000Z","title":"VideoMV: Consistent Multi-View Generation Based on Large Video\n  Generative Model","summary":"Generating multi-view images based on text or single-image prompts is a\ncritical capability for the creation of 3D content. Two fundamental questions\non this topic are what data we use for training and how to ensure multi-view\nconsistency. This paper introduces a novel framework that makes fundamental\ncontributions to both questions. Unlike leveraging images from 2D diffusion\nmodels for training, we propose a dense consistent multi-view generation model\nthat is fine-tuned from off-the-shelf video generative models. Images from\nvideo generative models are more suitable for multi-view generation because the\nunderlying network architecture that generates them employs a temporal module\nto enforce frame consistency. Moreover, the video data sets used to train these\nmodels are abundant and diverse, leading to a reduced train-finetuning domain\ngap. To enhance multi-view consistency, we introduce a 3D-Aware Denoising\nSampling, which first employs a feed-forward reconstruction module to get an\nexplicit global 3D model, and then adopts a sampling strategy that effectively\ninvolves images rendered from the global 3D model into the denoising sampling\nloop to improve the multi-view consistency of the final images. As a\nby-product, this module also provides a fast way to create 3D assets\nrepresented by 3D Gaussians within a few seconds. Our approach can generate 24\ndense views and converges much faster in training than state-of-the-art\napproaches (4 GPU hours versus many thousand GPU hours) with comparable visual\nquality and consistency. By further fine-tuning, our approach outperforms\nexisting state-of-the-art methods in both quantitative metrics and visual\neffects. Our project page is aigc3d.github.io/VideoMV.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2403.12010.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"VideoMV: Consistent Multi-View Generation Based on Large Video\n  Generative Model"}],"highlightedSummary":[{"type":"highlight","text":"Generating multi-view images based on text or single-image prompts is a\ncritical capability for the creation of 3D content."},{"type":"text","text":" Two fundamental questions\non this topic are what data we use for training and how to ensure multi-view\nconsistency. This paper introduces a novel framework that makes fundamental\ncontributions to both questions. Unlike leveraging images from 2D diffusion\nmodels for training, we propose a dense consistent multi-view generation model\nthat is fine-tuned from off-the-shelf video generative models. Images from\nvideo generative models are more suitable for multi-view generation because the\nunderlying network architecture that generates them employs a temporal module\nto enforce frame consistency. Moreover, the video data sets used to train these\nmodels are abundant and diverse, leading to a reduced train-finetuning domain\ngap. To enhance multi-view consistency, we introduce a 3D-Aware Denoising\nSampling, which first employs a feed-forward reconstruction module to get an\nexplicit global 3D model, and then adopts a sampling strategy that effectively\ninvolves images rendered from the global 3D model into the denoising sampling\nloop to improve the multi-view consistency of the final images. As a\nby-product, this module also provides a fast way to create 3D assets\nrepresented by 3D Gaussians within a few seconds. Our approach can generate 24\ndense views and converges much faster in training than state-of-the-art\napproaches (4 GPU hours versus many thousand GPU hours) with comparable visual\nquality and consistency. By further fine-tuning, our approach outperforms\nexisting state-of-the-art methods in both quantitative metrics and visual\neffects. Our project page is aigc3d.github.io/VideoMV."}]},{"paper":{"id":"2601.00678","authors":[{"_id":"695dda50c03d6d81e4399fa3","name":"Melonie de Almeida","hidden":false},{"_id":"695dda50c03d6d81e4399fa4","name":"Daniela Ivanova","hidden":false},{"_id":"695dda50c03d6d81e4399fa5","name":"Tong Shi","hidden":false},{"_id":"695dda50c03d6d81e4399fa6","name":"John H. Williamson","hidden":false},{"_id":"695dda50c03d6d81e4399fa7","name":"Paul Henderson","hidden":false}],"publishedAt":"2026-01-02T13:04:47.000Z","title":"Pixel-to-4D: Camera-Controlled Image-to-Video Generation with Dynamic 3D Gaussians","summary":"Humans excel at forecasting the future dynamics of a scene given just a single image. Video generation models that can mimic this ability are an essential component for intelligent systems. Recent approaches have improved temporal coherence and 3D consistency in single-image-conditioned video generation. However, these methods often lack robust user controllability, such as modifying the camera path, limiting their applicability in real-world applications. Most existing camera-controlled image-to-video models struggle with accurately modeling camera motion, maintaining temporal consistency, and preserving geometric integrity. Leveraging explicit intermediate 3D representations offers a promising solution by enabling coherent video generation aligned with a given camera trajectory. Although these methods often use 3D point clouds to render scenes and introduce object motion in a later stage, this two-step process still falls short in achieving full temporal consistency, despite allowing precise control over camera movement. We propose a novel framework that constructs a 3D Gaussian scene representation and samples plausible object motion, given a single image in a single forward pass. This enables fast, camera-guided video generation without the need for iterative denoising to inject object motion into render frames. Extensive experiments on the KITTI, Waymo, RealEstate10K and DL3DV-10K datasets demonstrate that our method achieves state-of-the-art video quality and inference efficiency. The project page is available at https://melonienimasha.github.io/Pixel-to-4D-Website.","upvotes":1,"discussionId":"695dda50c03d6d81e4399fa8","projectPage":"https://melonienimasha.github.io/Pixel-to-4D-Website/","ai_summary":"A novel framework generates camera-guided videos from single images using 3D Gaussian scene representation and simultaneous object motion sampling for improved temporal consistency and efficiency.","ai_keywords":["video generation","3D Gaussian scene representation","camera trajectory","temporal consistency","object motion sampling","single-image-conditioned video generation","explicit intermediate 3D representations","inference efficiency"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2026-01-02T08:04:47.000Z","title":"Pixel-to-4D: Camera-Controlled Image-to-Video Generation with Dynamic 3D Gaussians","summary":"Humans excel at forecasting the future dynamics of a scene given just a single image. Video generation models that can mimic this ability are an essential component for intelligent systems. Recent approaches have improved temporal coherence and 3D consistency in single-image-conditioned video generation. However, these methods often lack robust user controllability, such as modifying the camera path, limiting their applicability in real-world applications. Most existing camera-controlled image-to-video models struggle with accurately modeling camera motion, maintaining temporal consistency, and preserving geometric integrity. Leveraging explicit intermediate 3D representations offers a promising solution by enabling coherent video generation aligned with a given camera trajectory. Although these methods often use 3D point clouds to render scenes and introduce object motion in a later stage, this two-step process still falls short in achieving full temporal consistency, despite allowing precise control over camera movement. We propose a novel framework that constructs a 3D Gaussian scene representation and samples plausible object motion, given a single image in a single forward pass. This enables fast, camera-guided video generation without the need for iterative denoising to inject object motion into render frames. Extensive experiments on the KITTI, Waymo, RealEstate10K and DL3DV-10K datasets demonstrate that our method achieves state-of-the-art video quality and inference efficiency. The project page is available at https://melonienimasha.github.io/Pixel-to-4D-Website.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2601.00678.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Pixel-to-4D: Camera-Controlled Image-to-Video Generation with Dynamic 3D Gaussians"}],"highlightedSummary":[{"type":"text","text":"Humans excel at forecasting the future dynamics of a scene given just a single image. Video generation models that can mimic this ability are an essential component for intelligent systems. Recent approaches have improved temporal coherence and 3D consistency in single-image-conditioned video generation. However, these methods often lack robust user controllability, such as modifying the camera path, limiting their applicability in real-world applications. Most existing camera-controlled image-to-video models struggle with accurately modeling camera motion, maintaining temporal consistency, and preserving geometric integrity. Leveraging explicit intermediate 3D representations offers a promising solution by enabling coherent video generation aligned with a given camera trajectory. Although these methods often use 3D point clouds to render scenes and introduce object motion in a later stage, this two-step process still falls short in achieving full temporal consistency, despite allowing precise control over camera movement. "},{"type":"highlight","text":"We propose a novel framework that constructs a 3D Gaussian scene representation and samples plausible object motion, given a single image in a single forward pass."},{"type":"text","text":" This enables fast, camera-guided video generation without the need for iterative denoising to inject object motion into render frames. Extensive experiments on the KITTI, Waymo, RealEstate10K and DL3DV-10K datasets demonstrate that our method achieves state-of-the-art video quality and inference efficiency. The project page is available at https://melonienimasha.github.io/Pixel-to-4D-Website."}]},{"paper":{"id":"2404.07199","authors":[{"_id":"66173de961c1ebcda969238b","user":{"_id":"6109b9266bb9e045866e3329","avatarUrl":"/avatars/caade782fd0ecc5a0f3b0bbbc7d16ccb.svg","isPro":false,"fullname":"Jaidev Shriram","user":"JaidevShriram","type":"user","name":"JaidevShriram"},"name":"Jaidev Shriram","status":"admin_assigned","statusLastChangedAt":"2024-04-18T13:15:53.741Z","hidden":false},{"_id":"66173de961c1ebcda969238c","user":{"_id":"656a76839dcedd16d5d4ed49","avatarUrl":"/avatars/81e4a17f9af8c990dc13bf1a7bb4efbd.svg","isPro":false,"fullname":"Alex Trevithick","user":"atrevithick","type":"user","name":"atrevithick"},"name":"Alex Trevithick","status":"admin_assigned","statusLastChangedAt":"2024-04-18T13:15:59.848Z","hidden":false},{"_id":"66173de961c1ebcda969238d","user":{"_id":"65ef493cd08d92badad38e54","avatarUrl":"/avatars/734fde502646760eab2c82ea87bfdff0.svg","isPro":false,"fullname":"liulingie","user":"liulingjie","type":"user","name":"liulingjie"},"name":"Lingjie Liu","status":"admin_assigned","statusLastChangedAt":"2024-04-18T13:16:14.360Z","hidden":false},{"_id":"66173de961c1ebcda969238e","name":"Ravi Ramamoorthi","hidden":false}],"publishedAt":"2024-04-10T17:57:41.000Z","title":"RealmDreamer: Text-Driven 3D Scene Generation with Inpainting and Depth\n  Diffusion","summary":"We introduce RealmDreamer, a technique for generation of general\nforward-facing 3D scenes from text descriptions. Our technique optimizes a 3D\nGaussian Splatting representation to match complex text prompts. We initialize\nthese splats by utilizing the state-of-the-art text-to-image generators,\nlifting their samples into 3D, and computing the occlusion volume. We then\noptimize this representation across multiple views as a 3D inpainting task with\nimage-conditional diffusion models. To learn correct geometric structure, we\nincorporate a depth diffusion model by conditioning on the samples from the\ninpainting model, giving rich geometric structure. Finally, we finetune the\nmodel using sharpened samples from image generators. Notably, our technique\ndoes not require video or multi-view data and can synthesize a variety of\nhigh-quality 3D scenes in different styles, consisting of multiple objects. Its\ngenerality additionally allows 3D synthesis from a single image.","upvotes":27,"discussionId":"66173dec61c1ebcda969244f","projectPage":"https://realmdreamer.github.io/","githubRepo":"https://github.com/jaidevshriram/realmdreamer","githubRepoAddedBy":"auto","ai_summary":"RealmDreamer generates 3D scenes from text descriptions using 3D Gaussian Splatting, image-conditional diffusion models, and depth diffusion models, achieving high-quality and diverse scene synthesis without requiring video or multi-view data.","ai_keywords":["3D Gaussian Splatting","text-to-image generators","image-conditional diffusion models","depth diffusion models","3D inpainting task","sharpened samples"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":303},"publishedAt":"2024-04-10T13:57:41.000Z","title":"RealmDreamer: Text-Driven 3D Scene Generation with Inpainting and Depth\n  Diffusion","summary":"We introduce RealmDreamer, a technique for generation of general\nforward-facing 3D scenes from text descriptions. Our technique optimizes a 3D\nGaussian Splatting representation to match complex text prompts. We initialize\nthese splats by utilizing the state-of-the-art text-to-image generators,\nlifting their samples into 3D, and computing the occlusion volume. We then\noptimize this representation across multiple views as a 3D inpainting task with\nimage-conditional diffusion models. To learn correct geometric structure, we\nincorporate a depth diffusion model by conditioning on the samples from the\ninpainting model, giving rich geometric structure. Finally, we finetune the\nmodel using sharpened samples from image generators. Notably, our technique\ndoes not require video or multi-view data and can synthesize a variety of\nhigh-quality 3D scenes in different styles, consisting of multiple objects. Its\ngenerality additionally allows 3D synthesis from a single image.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2404.07199.png","numComments":3,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"RealmDreamer: Text-Driven 3D Scene Generation with Inpainting and Depth\n  Diffusion"}],"highlightedSummary":[{"type":"text","text":"We introduce RealmDreamer, a technique for generation of general\nforward-facing 3D scenes from text descriptions. Our technique optimizes a 3D\nGaussian Splatting representation to match complex text prompts. We initialize\nthese splats by utilizing the state-of-the-art text-to-image generators,\nlifting their samples into 3D, and computing the occlusion volume. We then\noptimize this representation across multiple views as a 3D inpainting task with\nimage-conditional diffusion models. To learn correct geometric structure, we\nincorporate a depth diffusion model by conditioning on the samples from the\ninpainting model, giving rich geometric structure. Finally, we finetune the\nmodel using sharpened samples from image generators. "},{"type":"highlight","text":"Notably, our technique\ndoes not require video or multi-view data and can synthesize a variety of\nhigh-quality 3D scenes in different styles, consisting of multiple objects."},{"type":"text","text":" Its\ngenerality additionally allows 3D synthesis from a single image."}]},{"paper":{"id":"2304.10700","authors":[{"_id":"64f5500c13fe87b1db98a1d0","name":"Jason J. Yu","hidden":false},{"_id":"64f5500c13fe87b1db98a1d1","name":"Fereshteh Forghani","hidden":false},{"_id":"64f5500c13fe87b1db98a1d2","name":"Konstantinos G. Derpanis","hidden":false},{"_id":"64f5500c13fe87b1db98a1d3","name":"Marcus A. Brubaker","hidden":false}],"publishedAt":"2023-04-21T02:01:02.000Z","title":"Long-Term Photometric Consistent Novel View Synthesis with Diffusion\n  Models","summary":"Novel view synthesis from a single input image is a challenging task, where\nthe goal is to generate a new view of a scene from a desired camera pose that\nmay be separated by a large motion. The highly uncertain nature of this\nsynthesis task due to unobserved elements within the scene (i.e. occlusion) and\noutside the field-of-view makes the use of generative models appealing to\ncapture the variety of possible outputs. In this paper, we propose a novel\ngenerative model capable of producing a sequence of photorealistic images\nconsistent with a specified camera trajectory, and a single starting image. Our\napproach is centred on an autoregressive conditional diffusion-based model\ncapable of interpolating visible scene elements, and extrapolating unobserved\nregions in a view, in a geometrically consistent manner. Conditioning is\nlimited to an image capturing a single camera view and the (relative) pose of\nthe new camera view. To measure the consistency over a sequence of generated\nviews, we introduce a new metric, the thresholded symmetric epipolar distance\n(TSED), to measure the number of consistent frame pairs in a sequence. While\nprevious methods have been shown to produce high quality images and consistent\nsemantics across pairs of views, we show empirically with our metric that they\nare often inconsistent with the desired camera poses. In contrast, we\ndemonstrate that our method produces both photorealistic and view-consistent\nimagery.","upvotes":0,"discussionId":"64f5500d13fe87b1db98a201","githubRepo":"https://github.com/YorkUCVIL/Photoconsistent-NVS","githubRepoAddedBy":"auto","ai_summary":"A novel autoregressive conditional diffusion-based model generates photorealistic and view-consistent image sequences from a single starting image and camera trajectory.","ai_keywords":["autoregressive conditional diffusion","photorealistic images","view consistency","thresholded symmetric epipolar distance","camera trajectory"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":45},"publishedAt":"2023-04-20T22:01:02.000Z","title":"Long-Term Photometric Consistent Novel View Synthesis with Diffusion\n  Models","summary":"Novel view synthesis from a single input image is a challenging task, where\nthe goal is to generate a new view of a scene from a desired camera pose that\nmay be separated by a large motion. The highly uncertain nature of this\nsynthesis task due to unobserved elements within the scene (i.e. occlusion) and\noutside the field-of-view makes the use of generative models appealing to\ncapture the variety of possible outputs. In this paper, we propose a novel\ngenerative model capable of producing a sequence of photorealistic images\nconsistent with a specified camera trajectory, and a single starting image. Our\napproach is centred on an autoregressive conditional diffusion-based model\ncapable of interpolating visible scene elements, and extrapolating unobserved\nregions in a view, in a geometrically consistent manner. Conditioning is\nlimited to an image capturing a single camera view and the (relative) pose of\nthe new camera view. To measure the consistency over a sequence of generated\nviews, we introduce a new metric, the thresholded symmetric epipolar distance\n(TSED), to measure the number of consistent frame pairs in a sequence. While\nprevious methods have been shown to produce high quality images and consistent\nsemantics across pairs of views, we show empirically with our metric that they\nare often inconsistent with the desired camera poses. In contrast, we\ndemonstrate that our method produces both photorealistic and view-consistent\nimagery.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2304.10700.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Long-Term Photometric Consistent Novel View Synthesis with Diffusion\n  Models"}],"highlightedSummary":[{"type":"text","text":"Novel view synthesis from a single input image is a challenging task, where\nthe goal is to generate a new view of a scene from a desired camera pose that\nmay be separated by a large motion. The highly uncertain nature of this\nsynthesis task due to unobserved elements within the scene (i.e. occlusion) and\noutside the field-of-view makes the use of generative models appealing to\ncapture the variety of possible outputs. "},{"type":"highlight","text":"In this paper, we propose a novel\ngenerative model capable of producing a sequence of photorealistic images\nconsistent with a specified camera trajectory, and a single starting image."},{"type":"text","text":" Our\napproach is centred on an autoregressive conditional diffusion-based model\ncapable of interpolating visible scene elements, and extrapolating unobserved\nregions in a view, in a geometrically consistent manner. Conditioning is\nlimited to an image capturing a single camera view and the (relative) pose of\nthe new camera view. To measure the consistency over a sequence of generated\nviews, we introduce a new metric, the thresholded symmetric epipolar distance\n(TSED), to measure the number of consistent frame pairs in a sequence. While\nprevious methods have been shown to produce high quality images and consistent\nsemantics across pairs of views, we show empirically with our metric that they\nare often inconsistent with the desired camera poses. In contrast, we\ndemonstrate that our method produces both photorealistic and view-consistent\nimagery."}]},{"paper":{"id":"2303.14167","authors":[{"_id":"64f464429e26f30d73f93e0a","name":"Yuanbo Yang","hidden":false},{"_id":"64f464429e26f30d73f93e0b","name":"Yifei Yang","hidden":false},{"_id":"64f464429e26f30d73f93e0c","name":"Hanlei Guo","hidden":false},{"_id":"64f464429e26f30d73f93e0d","name":"Rong Xiong","hidden":false},{"_id":"64f464429e26f30d73f93e0e","name":"Yue Wang","hidden":false},{"_id":"64f464429e26f30d73f93e0f","name":"Yiyi Liao","hidden":false}],"publishedAt":"2023-03-24T17:28:07.000Z","title":"UrbanGIRAFFE: Representing Urban Scenes as Compositional Generative\n  Neural Feature Fields","summary":"Generating photorealistic images with controllable camera pose and scene\ncontents is essential for many applications including AR/VR and simulation.\nDespite the fact that rapid progress has been made in 3D-aware generative\nmodels, most existing methods focus on object-centric images and are not\napplicable to generating urban scenes for free camera viewpoint control and\nscene editing. To address this challenging task, we propose UrbanGIRAFFE, which\nuses a coarse 3D panoptic prior, including the layout distribution of\nuncountable stuff and countable objects, to guide a 3D-aware generative model.\nOur model is compositional and controllable as it breaks down the scene into\nstuff, objects, and sky. Using stuff prior in the form of semantic voxel grids,\nwe build a conditioned stuff generator that effectively incorporates the coarse\nsemantic and geometry information. The object layout prior further allows us to\nlearn an object generator from cluttered scenes. With proper loss functions,\nour approach facilitates photorealistic 3D-aware image synthesis with diverse\ncontrollability, including large camera movement, stuff editing, and object\nmanipulation. We validate the effectiveness of our model on both synthetic and\nreal-world datasets, including the challenging KITTI-360 dataset.","upvotes":0,"discussionId":"64f464439e26f30d73f93e33","githubRepo":"https://github.com/freemty/urbangiraffe","githubRepoAddedBy":"auto","ai_summary":"The proposed UrbanGIRAFFE model facilitates photorealistic 3D-aware image synthesis with controllable camera pose and scene editing by using a coarse 3D panoptic prior and a compositional generative approach.","ai_keywords":["3D-aware generative models","panoptic prior","semantic voxel grids","stuff generator","object generator","photorealistic image synthesis","camera pose control","scene editing","KITTI-360 dataset"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":58},"publishedAt":"2023-03-24T13:28:07.000Z","title":"UrbanGIRAFFE: Representing Urban Scenes as Compositional Generative\n  Neural Feature Fields","summary":"Generating photorealistic images with controllable camera pose and scene\ncontents is essential for many applications including AR/VR and simulation.\nDespite the fact that rapid progress has been made in 3D-aware generative\nmodels, most existing methods focus on object-centric images and are not\napplicable to generating urban scenes for free camera viewpoint control and\nscene editing. To address this challenging task, we propose UrbanGIRAFFE, which\nuses a coarse 3D panoptic prior, including the layout distribution of\nuncountable stuff and countable objects, to guide a 3D-aware generative model.\nOur model is compositional and controllable as it breaks down the scene into\nstuff, objects, and sky. Using stuff prior in the form of semantic voxel grids,\nwe build a conditioned stuff generator that effectively incorporates the coarse\nsemantic and geometry information. The object layout prior further allows us to\nlearn an object generator from cluttered scenes. With proper loss functions,\nour approach facilitates photorealistic 3D-aware image synthesis with diverse\ncontrollability, including large camera movement, stuff editing, and object\nmanipulation. We validate the effectiveness of our model on both synthetic and\nreal-world datasets, including the challenging KITTI-360 dataset.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2303.14167.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"UrbanGIRAFFE: Representing Urban Scenes as Compositional Generative\n  Neural Feature Fields"}],"highlightedSummary":[{"type":"text","text":"Generating photorealistic images with controllable camera pose and scene\ncontents is essential for many applications including AR/VR and simulation.\n"},{"type":"highlight","text":"Despite the fact that rapid progress has been made in 3D-aware generative\nmodels, most existing methods focus on object-centric images and are not\napplicable to generating urban scenes for free camera viewpoint control and\nscene editing."},{"type":"text","text":" To address this challenging task, we propose UrbanGIRAFFE, which\nuses a coarse 3D panoptic prior, including the layout distribution of\nuncountable stuff and countable objects, to guide a 3D-aware generative model.\nOur model is compositional and controllable as it breaks down the scene into\nstuff, objects, and sky. Using stuff prior in the form of semantic voxel grids,\nwe build a conditioned stuff generator that effectively incorporates the coarse\nsemantic and geometry information. The object layout prior further allows us to\nlearn an object generator from cluttered scenes. With proper loss functions,\nour approach facilitates photorealistic 3D-aware image synthesis with diverse\ncontrollability, including large camera movement, stuff editing, and object\nmanipulation. We validate the effectiveness of our model on both synthetic and\nreal-world datasets, including the challenging KITTI-360 dataset."}]},{"paper":{"id":"2406.13099","authors":[{"_id":"667609215eaa9dd299f4feee","user":{"_id":"66749366b82e48404d157a66","avatarUrl":"/avatars/a56066447b0ebcc26c31ed2388a08707.svg","isPro":false,"fullname":"Paul Henderson","user":"pmh47","type":"user","name":"pmh47"},"name":"Paul Henderson","status":"claimed_verified","statusLastChangedAt":"2024-06-24T07:30:00.485Z","hidden":false},{"_id":"667609215eaa9dd299f4feef","name":"Melonie de Almeida","hidden":false},{"_id":"667609215eaa9dd299f4fef0","name":"Daniela Ivanova","hidden":false},{"_id":"667609215eaa9dd299f4fef1","name":"Titas Anciukevičius","hidden":false}],"publishedAt":"2024-06-18T23:14:29.000Z","title":"Sampling 3D Gaussian Scenes in Seconds with Latent Diffusion Models","summary":"We present a latent diffusion model over 3D scenes, that can be trained using\nonly 2D image data. To achieve this, we first design an autoencoder that maps\nmulti-view images to 3D Gaussian splats, and simultaneously builds a compressed\nlatent representation of these splats. Then, we train a multi-view diffusion\nmodel over the latent space to learn an efficient generative model. This\npipeline does not require object masks nor depths, and is suitable for complex\nscenes with arbitrary camera positions. We conduct careful experiments on two\nlarge-scale datasets of complex real-world scenes -- MVImgNet and\nRealEstate10K. We show that our approach enables generating 3D scenes in as\nlittle as 0.2 seconds, either from scratch, from a single input view, or from\nsparse input views. It produces diverse and high-quality results while running\nan order of magnitude faster than non-latent diffusion models and earlier\nNeRF-based generative models","upvotes":4,"discussionId":"667609255eaa9dd299f5002f","ai_summary":"A latent diffusion model capable of generating 3D scenes from 2D images enables fast and high-quality scene synthesis without masks or depth data.","ai_keywords":["latent diffusion model","multi-view images","3D Gaussian splats","compressed latent representation","multi-view diffusion model","latent space","MVImgNet","RealEstate10K","scene synthesis","generative model","NeRF-based generative models"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-06-18T19:14:29.000Z","title":"Sampling 3D Gaussian Scenes in Seconds with Latent Diffusion Models","summary":"We present a latent diffusion model over 3D scenes, that can be trained using\nonly 2D image data. To achieve this, we first design an autoencoder that maps\nmulti-view images to 3D Gaussian splats, and simultaneously builds a compressed\nlatent representation of these splats. Then, we train a multi-view diffusion\nmodel over the latent space to learn an efficient generative model. This\npipeline does not require object masks nor depths, and is suitable for complex\nscenes with arbitrary camera positions. We conduct careful experiments on two\nlarge-scale datasets of complex real-world scenes -- MVImgNet and\nRealEstate10K. We show that our approach enables generating 3D scenes in as\nlittle as 0.2 seconds, either from scratch, from a single input view, or from\nsparse input views. It produces diverse and high-quality results while running\nan order of magnitude faster than non-latent diffusion models and earlier\nNeRF-based generative models","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2406.13099.png","numComments":1,"upvoted":false,"isAuthorParticipating":true,"highlightedTitle":[{"type":"text","text":"Sampling 3D Gaussian Scenes in Seconds with Latent Diffusion Models"}],"highlightedSummary":[{"type":"text","text":"We present a latent diffusion model over 3D scenes, that can be trained using\nonly 2D image data. To achieve this, we first design an autoencoder that maps\nmulti-view images to 3D Gaussian splats, and simultaneously builds a compressed\nlatent representation of these splats. Then, we train a multi-view diffusion\nmodel over the latent space to learn an efficient generative model. This\npipeline does not require object masks nor depths, and is suitable for complex\nscenes with arbitrary camera positions. We conduct careful experiments on two\nlarge-scale datasets of complex real-world scenes -- MVImgNet and\nRealEstate10K. "},{"type":"highlight","text":"We show that our approach enables generating 3D scenes in as\nlittle as 0.2 seconds, either from scratch, from a single input view, or from\nsparse input views."},{"type":"text","text":" It produces diverse and high-quality results while running\nan order of magnitude faster than non-latent diffusion models and earlier\nNeRF-based generative models"}]},{"paper":{"id":"2411.18644","authors":[{"_id":"678f0e848af7a3998769fcd8","name":"Zhaofang Qian","hidden":false},{"_id":"678f0e848af7a3998769fcd9","name":"Abolfazl Sharifi","hidden":false},{"_id":"678f0e848af7a3998769fcda","name":"Tucker Carroll","hidden":false},{"_id":"678f0e848af7a3998769fcdb","name":"Ser-Nam Lim","hidden":false}],"publishedAt":"2024-11-26T19:21:57.000Z","title":"Scene Co-pilot: Procedural Text to Video Generation with Human in the\n  Loop","summary":"Video generation has achieved impressive quality, but it still suffers from\nartifacts such as temporal inconsistency and violation of physical laws.\nLeveraging 3D scenes can fundamentally resolve these issues by providing\nprecise control over scene entities. To facilitate the easy generation of\ndiverse photorealistic scenes, we propose Scene Copilot, a framework combining\nlarge language models (LLMs) with a procedural 3D scene generator.\nSpecifically, Scene Copilot consists of Scene Codex, BlenderGPT, and Human in\nthe loop. Scene Codex is designed to translate textual user input into commands\nunderstandable by the 3D scene generator. BlenderGPT provides users with an\nintuitive and direct way to precisely control the generated 3D scene and the\nfinal output video. Furthermore, users can utilize Blender UI to receive\ninstant visual feedback. Additionally, we have curated a procedural dataset of\nobjects in code format to further enhance our system's capabilities. Each\ncomponent works seamlessly together to support users in generating desired 3D\nscenes. Extensive experiments demonstrate the capability of our framework in\ncustomizing 3D scenes and video generation.","upvotes":0,"discussionId":"678f0e898af7a3998769feb5","ai_summary":"Scene Copilot, combining large language models and a procedural 3D scene generator, enhances video generation by providing precise control and reducing artifacts through 3D scene manipulation.","ai_keywords":["large language models","procedural 3D scene generator","text-to-command translation","BlenderGPT","user feedback","procedural dataset","3D scene customization","video generation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-11-26T14:21:57.000Z","title":"Scene Co-pilot: Procedural Text to Video Generation with Human in the\n  Loop","summary":"Video generation has achieved impressive quality, but it still suffers from\nartifacts such as temporal inconsistency and violation of physical laws.\nLeveraging 3D scenes can fundamentally resolve these issues by providing\nprecise control over scene entities. To facilitate the easy generation of\ndiverse photorealistic scenes, we propose Scene Copilot, a framework combining\nlarge language models (LLMs) with a procedural 3D scene generator.\nSpecifically, Scene Copilot consists of Scene Codex, BlenderGPT, and Human in\nthe loop. Scene Codex is designed to translate textual user input into commands\nunderstandable by the 3D scene generator. BlenderGPT provides users with an\nintuitive and direct way to precisely control the generated 3D scene and the\nfinal output video. Furthermore, users can utilize Blender UI to receive\ninstant visual feedback. Additionally, we have curated a procedural dataset of\nobjects in code format to further enhance our system's capabilities. Each\ncomponent works seamlessly together to support users in generating desired 3D\nscenes. Extensive experiments demonstrate the capability of our framework in\ncustomizing 3D scenes and video generation.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2411.18644.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Scene Co-pilot: Procedural Text to Video Generation with Human in the\n  Loop"}],"highlightedSummary":[{"type":"text","text":"Video generation has achieved impressive quality, but it still suffers from\nartifacts such as temporal inconsistency and violation of physical laws.\nLeveraging 3D scenes can fundamentally resolve these issues by providing\nprecise control over scene entities. "},{"type":"highlight","text":"To facilitate the easy generation of\ndiverse photorealistic scenes, we propose Scene Copilot, a framework combining\nlarge language models (LLMs) with a procedural 3D scene generator."},{"type":"text","text":"\nSpecifically, Scene Copilot consists of Scene Codex, BlenderGPT, and Human in\nthe loop. Scene Codex is designed to translate textual user input into commands\nunderstandable by the 3D scene generator. BlenderGPT provides users with an\nintuitive and direct way to precisely control the generated 3D scene and the\nfinal output video. Furthermore, users can utilize Blender UI to receive\ninstant visual feedback. Additionally, we have curated a procedural dataset of\nobjects in code format to further enhance our system's capabilities. Each\ncomponent works seamlessly together to support users in generating desired 3D\nscenes. Extensive experiments demonstrate the capability of our framework in\ncustomizing 3D scenes and video generation."}]},{"paper":{"id":"2304.02602","authors":[{"_id":"64f463cdbca5721430f56d63","name":"Eric R. Chan","hidden":false},{"_id":"64f463cdbca5721430f56d64","name":"Koki Nagano","hidden":false},{"_id":"64f463cdbca5721430f56d65","name":"Matthew A. Chan","hidden":false},{"_id":"64f463cdbca5721430f56d66","name":"Alexander W. Bergman","hidden":false},{"_id":"64f463cdbca5721430f56d67","user":{"_id":"6517d7f84fadbeb643c68269","avatarUrl":"/avatars/ad21288123abcd4488ae1bdcb10b54e7.svg","isPro":false,"fullname":"JJ Park","user":"jjparkcv","type":"user","name":"jjparkcv"},"name":"Jeong Joon Park","status":"claimed_verified","statusLastChangedAt":"2024-07-11T17:34:58.825Z","hidden":false},{"_id":"64f463cdbca5721430f56d68","name":"Axel Levy","hidden":false},{"_id":"64f463cdbca5721430f56d69","name":"Miika Aittala","hidden":false},{"_id":"64f463cdbca5721430f56d6a","name":"Shalini De Mello","hidden":false},{"_id":"64f463cdbca5721430f56d6b","user":{"_id":"66587947b7d313010028ac02","avatarUrl":"/avatars/cb95f524dd1c80aa6ec8af977c5f2606.svg","isPro":false,"fullname":"Tero Karras","user":"tkarras","type":"user","name":"tkarras"},"name":"Tero Karras","status":"claimed_verified","statusLastChangedAt":"2024-05-31T06:55:04.627Z","hidden":false},{"_id":"64f463cdbca5721430f56d6c","name":"Gordon Wetzstein","hidden":false}],"publishedAt":"2023-04-05T17:15:47.000Z","title":"Generative Novel View Synthesis with 3D-Aware Diffusion Models","summary":"We present a diffusion-based model for 3D-aware generative novel view\nsynthesis from as few as a single input image. Our model samples from the\ndistribution of possible renderings consistent with the input and, even in the\npresence of ambiguity, is capable of rendering diverse and plausible novel\nviews. To achieve this, our method makes use of existing 2D diffusion backbones\nbut, crucially, incorporates geometry priors in the form of a 3D feature\nvolume. This latent feature field captures the distribution over possible scene\nrepresentations and improves our method's ability to generate view-consistent\nnovel renderings. In addition to generating novel views, our method has the\nability to autoregressively synthesize 3D-consistent sequences. We demonstrate\nstate-of-the-art results on synthetic renderings and room-scale scenes; we also\nshow compelling results for challenging, real-world objects.","upvotes":0,"discussionId":"64f463d1bca5721430f56ddf","ai_summary":"A diffusion-based model generates 3D-aware novel views from a single image by using 2D diffusion backbones supplemented with geometry priors in the form of a 3D feature volume, enabling both novel view synthesis and 3D-consistent sequences.","ai_keywords":["diffusion-based model","generative novel view synthesis","distribution of possible renderings","2D diffusion backbones","3D feature volume","latent feature field","scene representations","view-consistent novel renderings","3D-consistent sequences"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2023-04-05T13:15:47.000Z","title":"Generative Novel View Synthesis with 3D-Aware Diffusion Models","summary":"We present a diffusion-based model for 3D-aware generative novel view\nsynthesis from as few as a single input image. Our model samples from the\ndistribution of possible renderings consistent with the input and, even in the\npresence of ambiguity, is capable of rendering diverse and plausible novel\nviews. To achieve this, our method makes use of existing 2D diffusion backbones\nbut, crucially, incorporates geometry priors in the form of a 3D feature\nvolume. This latent feature field captures the distribution over possible scene\nrepresentations and improves our method's ability to generate view-consistent\nnovel renderings. In addition to generating novel views, our method has the\nability to autoregressively synthesize 3D-consistent sequences. We demonstrate\nstate-of-the-art results on synthetic renderings and room-scale scenes; we also\nshow compelling results for challenging, real-world objects.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2304.02602.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Generative Novel View Synthesis with 3D-Aware Diffusion Models"}],"highlightedSummary":[{"type":"text","text":"We present a diffusion-based model for 3D-aware generative novel view\nsynthesis from as few as a single input image. Our model samples from the\ndistribution of possible renderings consistent with the input and, even in the\npresence of ambiguity, is capable of rendering diverse and plausible novel\nviews. To achieve this, our method makes use of existing 2D diffusion backbones\nbut, crucially, incorporates geometry priors in the form of a 3D feature\nvolume. This latent feature field captures the distribution over possible scene\nrepresentations and improves our method's ability to generate view-consistent\nnovel renderings. In addition to generating novel views, our method has the\nability to autoregressively synthesize 3D-consistent sequences. "},{"type":"highlight","text":"We demonstrate\nstate-of-the-art results on synthetic renderings and room-scale scenes; we also\nshow compelling results for challenging, real-world objects."}]},{"paper":{"id":"2207.09686","authors":[{"_id":"669f50ff8c65c172c4dcc304","name":"Qianyi Wu","hidden":false},{"_id":"669f50ff8c65c172c4dcc305","user":{"_id":"6423f10a3fa01ecba6fca861","avatarUrl":"/avatars/e4b4c7dddc5cf962ade45d8bdfa10b1a.svg","isPro":false,"fullname":"Xian Liu","user":"alvinliu0","type":"user","name":"alvinliu0"},"name":"Xian Liu","status":"claimed_verified","statusLastChangedAt":"2024-10-30T09:56:52.847Z","hidden":false},{"_id":"669f50ff8c65c172c4dcc306","user":{"_id":"653862bdbe39573b3b247b44","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/653862bdbe39573b3b247b44/oMqOC4USbQlPX5HFLDXt6.jpeg","isPro":false,"fullname":"Donny Chen","user":"donydchen","type":"user","name":"donydchen"},"name":"Yuedong Chen","status":"claimed_verified","statusLastChangedAt":"2024-07-23T07:47:03.682Z","hidden":false},{"_id":"669f50ff8c65c172c4dcc307","name":"Kejie Li","hidden":false},{"_id":"669f50ff8c65c172c4dcc308","name":"Chuanxia Zheng","hidden":false},{"_id":"669f50ff8c65c172c4dcc309","name":"Jianfei Cai","hidden":false},{"_id":"669f50ff8c65c172c4dcc30a","name":"Jianmin Zheng","hidden":false}],"publishedAt":"2022-07-20T06:38:04.000Z","title":"Object-Compositional Neural Implicit Surfaces","summary":"The neural implicit representation has shown its effectiveness in novel view\nsynthesis and high-quality 3D reconstruction from multi-view images. However,\nmost approaches focus on holistic scene representation yet ignore individual\nobjects inside it, thus limiting potential downstream applications. In order to\nlearn object-compositional representation, a few works incorporate the 2D\nsemantic map as a cue in training to grasp the difference between objects. But\nthey neglect the strong connections between object geometry and instance\nsemantic information, which leads to inaccurate modeling of individual\ninstance. This paper proposes a novel framework, ObjectSDF, to build an\nobject-compositional neural implicit representation with high fidelity in 3D\nreconstruction and object representation. Observing the ambiguity of\nconventional volume rendering pipelines, we model the scene by combining the\nSigned Distance Functions (SDF) of individual object to exert explicit surface\nconstraint. The key in distinguishing different instances is to revisit the\nstrong association between an individual object's SDF and semantic label.\nParticularly, we convert the semantic information to a function of object SDF\nand develop a unified and compact representation for scene and objects.\nExperimental results show the superiority of ObjectSDF framework in\nrepresenting both the holistic object-compositional scene and the individual\ninstances. Code can be found at https://qianyiwu.github.io/objectsdf/","upvotes":0,"discussionId":"669f51018c65c172c4dcc3ba","projectPage":"https://wuqianyi.top/objectsdf/","githubRepo":"https://github.com/QianyiWu/objsdf","githubRepoAddedBy":"user","ai_summary":"The ObjectSDF framework enhances 3D reconstruction by integrating signed distance functions with semantic maps for accurate compositional representation of scenes and objects.","ai_keywords":["neural implicit representation","novel view synthesis","3D reconstruction","holistic scene representation","object-compositional representation","2D semantic map","signed distance functions (SDF)","volume rendering pipelines","semantic label","unified representation"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct","githubStars":191},"publishedAt":"2022-07-20T02:38:04.000Z","title":"Object-Compositional Neural Implicit Surfaces","summary":"The neural implicit representation has shown its effectiveness in novel view\nsynthesis and high-quality 3D reconstruction from multi-view images. However,\nmost approaches focus on holistic scene representation yet ignore individual\nobjects inside it, thus limiting potential downstream applications. In order to\nlearn object-compositional representation, a few works incorporate the 2D\nsemantic map as a cue in training to grasp the difference between objects. But\nthey neglect the strong connections between object geometry and instance\nsemantic information, which leads to inaccurate modeling of individual\ninstance. This paper proposes a novel framework, ObjectSDF, to build an\nobject-compositional neural implicit representation with high fidelity in 3D\nreconstruction and object representation. Observing the ambiguity of\nconventional volume rendering pipelines, we model the scene by combining the\nSigned Distance Functions (SDF) of individual object to exert explicit surface\nconstraint. The key in distinguishing different instances is to revisit the\nstrong association between an individual object's SDF and semantic label.\nParticularly, we convert the semantic information to a function of object SDF\nand develop a unified and compact representation for scene and objects.\nExperimental results show the superiority of ObjectSDF framework in\nrepresenting both the holistic object-compositional scene and the individual\ninstances. Code can be found at https://qianyiwu.github.io/objectsdf/","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2207.09686.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Object-Compositional Neural Implicit Surfaces"}],"highlightedSummary":[{"type":"text","text":"The neural implicit representation has shown its effectiveness in novel view\nsynthesis and high-quality 3D reconstruction from multi-view images. "},{"type":"highlight","text":"However,\nmost approaches focus on holistic scene representation yet ignore individual\nobjects inside it, thus limiting potential downstream applications."},{"type":"text","text":" In order to\nlearn object-compositional representation, a few works incorporate the 2D\nsemantic map as a cue in training to grasp the difference between objects. But\nthey neglect the strong connections between object geometry and instance\nsemantic information, which leads to inaccurate modeling of individual\ninstance. This paper proposes a novel framework, ObjectSDF, to build an\nobject-compositional neural implicit representation with high fidelity in 3D\nreconstruction and object representation. Observing the ambiguity of\nconventional volume rendering pipelines, we model the scene by combining the\nSigned Distance Functions (SDF) of individual object to exert explicit surface\nconstraint. The key in distinguishing different instances is to revisit the\nstrong association between an individual object's SDF and semantic label.\nParticularly, we convert the semantic information to a function of object SDF\nand develop a unified and compact representation for scene and objects.\nExperimental results show the superiority of ObjectSDF framework in\nrepresenting both the holistic object-compositional scene and the individual\ninstances. Code can be found at https://qianyiwu.github.io/objectsdf/"}]},{"paper":{"id":"2404.00345","authors":[{"_id":"660b93319d2edd43f73f32e1","name":"Takayuki Hara","hidden":false},{"_id":"660b93319d2edd43f73f32e2","name":"Tatsuya Harada","hidden":false}],"publishedAt":"2024-03-30T12:50:25.000Z","title":"MaGRITTe: Manipulative and Generative 3D Realization from Image, Topview\n  and Text","summary":"The generation of 3D scenes from user-specified conditions offers a promising\navenue for alleviating the production burden in 3D applications. Previous\nstudies required significant effort to realize the desired scene, owing to\nlimited control conditions. We propose a method for controlling and generating\n3D scenes under multimodal conditions using partial images, layout information\nrepresented in the top view, and text prompts. Combining these conditions to\ngenerate a 3D scene involves the following significant difficulties: (1) the\ncreation of large datasets, (2) reflection on the interaction of multimodal\nconditions, and (3) domain dependence of the layout conditions. We decompose\nthe process of 3D scene generation into 2D image generation from the given\nconditions and 3D scene generation from 2D images. 2D image generation is\nachieved by fine-tuning a pretrained text-to-image model with a small\nartificial dataset of partial images and layouts, and 3D scene generation is\nachieved by layout-conditioned depth estimation and neural radiance fields\n(NeRF), thereby avoiding the creation of large datasets. The use of a common\nrepresentation of spatial information using 360-degree images allows for the\nconsideration of multimodal condition interactions and reduces the domain\ndependence of the layout control. The experimental results qualitatively and\nquantitatively demonstrated that the proposed method can generate 3D scenes in\ndiverse domains, from indoor to outdoor, according to multimodal conditions.","upvotes":19,"discussionId":"660b93329d2edd43f73f334e","ai_summary":"A method for generating 3D scenes from multimodal conditions, including partial images, top-view layouts, and text prompts, employs text-to-image models and neural radiance fields to produce diverse 3D scenes with reduced dataset requirements.","ai_keywords":["text-to-image model","neural radiance fields","NeRF","3D scene generation","2D image generation","layout-conditioned depth estimation","360-degree images"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2024-03-30T08:50:25.000Z","title":"MaGRITTe: Manipulative and Generative 3D Realization from Image, Topview\n  and Text","summary":"The generation of 3D scenes from user-specified conditions offers a promising\navenue for alleviating the production burden in 3D applications. Previous\nstudies required significant effort to realize the desired scene, owing to\nlimited control conditions. We propose a method for controlling and generating\n3D scenes under multimodal conditions using partial images, layout information\nrepresented in the top view, and text prompts. Combining these conditions to\ngenerate a 3D scene involves the following significant difficulties: (1) the\ncreation of large datasets, (2) reflection on the interaction of multimodal\nconditions, and (3) domain dependence of the layout conditions. We decompose\nthe process of 3D scene generation into 2D image generation from the given\nconditions and 3D scene generation from 2D images. 2D image generation is\nachieved by fine-tuning a pretrained text-to-image model with a small\nartificial dataset of partial images and layouts, and 3D scene generation is\nachieved by layout-conditioned depth estimation and neural radiance fields\n(NeRF), thereby avoiding the creation of large datasets. The use of a common\nrepresentation of spatial information using 360-degree images allows for the\nconsideration of multimodal condition interactions and reduces the domain\ndependence of the layout control. The experimental results qualitatively and\nquantitatively demonstrated that the proposed method can generate 3D scenes in\ndiverse domains, from indoor to outdoor, according to multimodal conditions.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2404.00345.png","numComments":11,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"MaGRITTe: Manipulative and Generative 3D Realization from Image, Topview\n  and Text"}],"highlightedSummary":[{"type":"text","text":"The generation of 3D scenes from user-specified conditions offers a promising\navenue for alleviating the production burden in 3D applications. Previous\nstudies required significant effort to realize the desired scene, owing to\nlimited control conditions. We propose a method for controlling and generating\n3D scenes under multimodal conditions using partial images, layout information\nrepresented in the top view, and text prompts. Combining these conditions to\ngenerate a 3D scene involves the following significant difficulties: (1) the\ncreation of large datasets, (2) reflection on the interaction of multimodal\nconditions, and (3) domain dependence of the layout conditions. "},{"type":"highlight","text":"We decompose\nthe process of 3D scene generation into 2D image generation from the given\nconditions and 3D scene generation from 2D images."},{"type":"text","text":" 2D image generation is\nachieved by fine-tuning a pretrained text-to-image model with a small\nartificial dataset of partial images and layouts, and 3D scene generation is\nachieved by layout-conditioned depth estimation and neural radiance fields\n(NeRF), thereby avoiding the creation of large datasets. The use of a common\nrepresentation of spatial information using 360-degree images allows for the\nconsideration of multimodal condition interactions and reduces the domain\ndependence of the layout control. The experimental results qualitatively and\nquantitatively demonstrated that the proposed method can generate 3D scenes in\ndiverse domains, from indoor to outdoor, according to multimodal conditions."}]},{"paper":{"id":"2501.02519","authors":[{"_id":"677e2ea4057fab4e86e92c7a","name":"Minglin Chen","hidden":false},{"_id":"677e2ea4057fab4e86e92c7b","name":"Longguang Wang","hidden":false},{"_id":"677e2ea4057fab4e86e92c7c","name":"Sheng Ao","hidden":false},{"_id":"677e2ea4057fab4e86e92c7d","name":"Ye Zhang","hidden":false},{"_id":"677e2ea4057fab4e86e92c7e","name":"Kai Xu","hidden":false},{"_id":"677e2ea4057fab4e86e92c7f","name":"Yulan Guo","hidden":false}],"publishedAt":"2025-01-05T12:20:13.000Z","title":"Layout2Scene: 3D Semantic Layout Guided Scene Generation via Geometry\n  and Appearance Diffusion Priors","summary":"3D scene generation conditioned on text prompts has significantly progressed\ndue to the development of 2D diffusion generation models. However, the textual\ndescription of 3D scenes is inherently inaccurate and lacks fine-grained\ncontrol during training, leading to implausible scene generation. As an\nintuitive and feasible solution, the 3D layout allows for precise specification\nof object locations within the scene. To this end, we present a text-to-scene\ngeneration method (namely, Layout2Scene) using additional semantic layout as\nthe prompt to inject precise control of 3D object positions. Specifically, we\nfirst introduce a scene hybrid representation to decouple objects and\nbackgrounds, which is initialized via a pre-trained text-to-3D model. Then, we\npropose a two-stage scheme to optimize the geometry and appearance of the\ninitialized scene separately. To fully leverage 2D diffusion priors in geometry\nand appearance generation, we introduce a semantic-guided geometry diffusion\nmodel and a semantic-geometry guided diffusion model which are finetuned on a\nscene dataset. Extensive experiments demonstrate that our method can generate\nmore plausible and realistic scenes as compared to state-of-the-art approaches.\nFurthermore, the generated scene allows for flexible yet precise editing,\nthereby facilitating multiple downstream applications.","upvotes":1,"discussionId":"677e2ea6057fab4e86e92cf0","ai_summary":"Layout2Scene uses semantic layouts to enhance text-to-3D scene generation by optimizing geometry and appearance with semantic-guided diffusion models, improving realism and flexibility.","ai_keywords":["scene hybrid representation","text-to-3D model","two-stage scheme","semantic-guided geometry diffusion model","semantic-geometry guided diffusion model"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-01-05T07:20:13.000Z","title":"Layout2Scene: 3D Semantic Layout Guided Scene Generation via Geometry\n  and Appearance Diffusion Priors","summary":"3D scene generation conditioned on text prompts has significantly progressed\ndue to the development of 2D diffusion generation models. However, the textual\ndescription of 3D scenes is inherently inaccurate and lacks fine-grained\ncontrol during training, leading to implausible scene generation. As an\nintuitive and feasible solution, the 3D layout allows for precise specification\nof object locations within the scene. To this end, we present a text-to-scene\ngeneration method (namely, Layout2Scene) using additional semantic layout as\nthe prompt to inject precise control of 3D object positions. Specifically, we\nfirst introduce a scene hybrid representation to decouple objects and\nbackgrounds, which is initialized via a pre-trained text-to-3D model. Then, we\npropose a two-stage scheme to optimize the geometry and appearance of the\ninitialized scene separately. To fully leverage 2D diffusion priors in geometry\nand appearance generation, we introduce a semantic-guided geometry diffusion\nmodel and a semantic-geometry guided diffusion model which are finetuned on a\nscene dataset. Extensive experiments demonstrate that our method can generate\nmore plausible and realistic scenes as compared to state-of-the-art approaches.\nFurthermore, the generated scene allows for flexible yet precise editing,\nthereby facilitating multiple downstream applications.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2501.02519.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"Layout2Scene: 3D Semantic Layout Guided Scene Generation via Geometry\n  and Appearance Diffusion Priors"}],"highlightedSummary":[{"type":"text","text":"3D scene generation conditioned on text prompts has significantly progressed\ndue to the development of 2D diffusion generation models. However, the textual\ndescription of 3D scenes is inherently inaccurate and lacks fine-grained\ncontrol during training, leading to implausible scene generation. As an\nintuitive and feasible solution, the 3D layout allows for precise specification\nof object locations within the scene. To this end, we present a text-to-scene\ngeneration method (namely, Layout2Scene) using additional semantic layout as\nthe prompt to inject precise control of 3D object positions. "},{"type":"highlight","text":"Specifically, we\nfirst introduce a scene hybrid representation to decouple objects and\nbackgrounds, which is initialized via a pre-trained text-to-3D model."},{"type":"text","text":" Then, we\npropose a two-stage scheme to optimize the geometry and appearance of the\ninitialized scene separately. To fully leverage 2D diffusion priors in geometry\nand appearance generation, we introduce a semantic-guided geometry diffusion\nmodel and a semantic-geometry guided diffusion model which are finetuned on a\nscene dataset. Extensive experiments demonstrate that our method can generate\nmore plausible and realistic scenes as compared to state-of-the-art approaches.\nFurthermore, the generated scene allows for flexible yet precise editing,\nthereby facilitating multiple downstream applications."}]},{"paper":{"id":"2504.00992","authors":[{"_id":"67ed3f3e13f29b32b8614fb6","user":{"_id":"62df0ae635a8bfe88fc4ac80","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/62df0ae635a8bfe88fc4ac80/yWu5Kfq_88Ni2uy3HvnLe.jpeg","isPro":false,"fullname":"elisabetta fedele","user":"efedele","type":"user","name":"efedele"},"name":"Elisabetta Fedele","status":"claimed_verified","statusLastChangedAt":"2025-04-08T06:57:16.779Z","hidden":false},{"_id":"67ed3f3e13f29b32b8614fb7","name":"Boyang Sun","hidden":false},{"_id":"67ed3f3e13f29b32b8614fb8","name":"Leonidas Guibas","hidden":false},{"_id":"67ed3f3e13f29b32b8614fb9","user":{"_id":"67b5fa179782a5e2fd2cb26a","avatarUrl":"/avatars/62c38f29ec641e001eeddf840bea21a0.svg","isPro":false,"fullname":"Marc Pollefeys","user":"mapo1","type":"user","name":"mapo1"},"name":"Marc Pollefeys","status":"claimed_verified","statusLastChangedAt":"2025-07-09T15:11:02.156Z","hidden":false},{"_id":"67ed3f3e13f29b32b8614fba","user":{"_id":"671a7160e45d92e304e82f45","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/no-auth/-trHIg97qX0dE9T4UYycc.png","isPro":false,"fullname":"Francis Engelmann","user":"francisengelmann","type":"user","name":"francisengelmann"},"name":"Francis Engelmann","status":"claimed_verified","statusLastChangedAt":"2025-05-18T08:47:46.737Z","hidden":false}],"publishedAt":"2025-04-01T17:29:35.000Z","title":"SuperDec: 3D Scene Decomposition with Superquadric Primitives","summary":"We present SuperDec, an approach for creating compact 3D scene\nrepresentations via decomposition into superquadric primitives. While most\nrecent works leverage geometric primitives to obtain photorealistic 3D scene\nrepresentations, we propose to leverage them to obtain a compact yet expressive\nrepresentation. We propose to solve the problem locally on individual objects\nand leverage the capabilities of instance segmentation methods to scale our\nsolution to full 3D scenes. In doing that, we design a new architecture which\nefficiently decompose point clouds of arbitrary objects in a compact set of\nsuperquadrics. We train our architecture on ShapeNet and we prove its\ngeneralization capabilities on object instances extracted from the ScanNet++\ndataset as well as on full Replica scenes. Finally, we show how a compact\nrepresentation based on superquadrics can be useful for a diverse range of\ndownstream applications, including robotic tasks and controllable visual\ncontent generation and editing.","upvotes":0,"discussionId":"67ed3f4013f29b32b861500a","projectPage":"https://super-dec.github.io/","ai_summary":"SuperDec creates compact 3D scene representations using superquadric primitives for efficient point cloud decomposition, demonstrating generalization and utility in various applications.","ai_keywords":["superquadric primitives","point clouds","ShapeNet","ScanNet++","Replica scenes","compact representation","instance segmentation","decomposition","photorealistic 3D scene representations"],"ai_summary_model":"Qwen/Qwen2.5-Coder-32B-Instruct"},"publishedAt":"2025-04-01T13:29:35.000Z","title":"SuperDec: 3D Scene Decomposition with Superquadric Primitives","summary":"We present SuperDec, an approach for creating compact 3D scene\nrepresentations via decomposition into superquadric primitives. While most\nrecent works leverage geometric primitives to obtain photorealistic 3D scene\nrepresentations, we propose to leverage them to obtain a compact yet expressive\nrepresentation. We propose to solve the problem locally on individual objects\nand leverage the capabilities of instance segmentation methods to scale our\nsolution to full 3D scenes. In doing that, we design a new architecture which\nefficiently decompose point clouds of arbitrary objects in a compact set of\nsuperquadrics. We train our architecture on ShapeNet and we prove its\ngeneralization capabilities on object instances extracted from the ScanNet++\ndataset as well as on full Replica scenes. Finally, we show how a compact\nrepresentation based on superquadrics can be useful for a diverse range of\ndownstream applications, including robotic tasks and controllable visual\ncontent generation and editing.","thumbnail":"https://cdn-thumbnails.huggingface.co/social-thumbnails/papers/2504.00992.png","numComments":0,"upvoted":false,"isAuthorParticipating":false,"highlightedTitle":[{"type":"text","text":"SuperDec: 3D Scene Decomposition with Superquadric Primitives"}],"highlightedSummary":[{"type":"text","text":"We present SuperDec, an approach for creating compact 3D scene\nrepresentations via decomposition into superquadric primitives. While most\nrecent works leverage geometric primitives to obtain photorealistic 3D scene\nrepresentations, we propose to leverage them to obtain a compact yet expressive\nrepresentation. "},{"type":"highlight","text":"We propose to solve the problem locally on individual objects\nand leverage the capabilities of instance segmentation methods to scale our\nsolution to full 3D scenes."},{"type":"text","text":" In doing that, we design a new architecture which\nefficiently decompose point clouds of arbitrary objects in a compact set of\nsuperquadrics. We train our architecture on ShapeNet and we prove its\ngeneralization capabilities on object instances extracted from the ScanNet++\ndataset as well as on full Replica scenes. Finally, we show how a compact\nrepresentation based on superquadrics can be useful for a diverse range of\ndownstream applications, including robotic tasks and controllable visual\ncontent generation and editing."}]}]