使用Dino+SAM+Stable diffusion 自动进行图片的修改

SAM 是Mata发布的“Segment Anything Model”可以准确识别和提取图像中的对象。它可以分割任何的图片，但是如果需要分割特定的物体，则需要需要点、框的特定提示才能准确分割图像。所以本文将介绍一种称为 Grounding Dino 的技术来自动生成 SAM 进行分割所需的框。

除了分割以外，我们还可以通过将 SAM 与 Grounding Dino 和 Stable Diffusion 相结合，获得高度准确图像分割结果，并且对分割后的图像进行细微的更改。

下面就是我们需要的所有的包：

 `%cd /content!git clone https://github.com/IDEA-Research/Grounded-Segment-Anything%cd /content/Grounded-Segment-Anything !pip install -q-r requirements.txt%cd /content/Grounded-Segment-Anything/GroundingDINO!pip install -q .%cd /content/Grounded-Segment-Anything/segment_anything!pip install -q .%cd /content/Grounded-Segment-Anything

导入必要的包：

 importos, syssys.path.append(os.path.join(os.getcwd(), "GroundingDINO"))importargparseimportcopyfromIPython.displayimportdisplayfromPILimportImage, ImageDraw, ImageFontfromtorchvision.opsimportbox_convert# Grounding DINOimportGroundingDINO.groundingdino.datasets.transformsasTfromGroundingDINO.groundingdino.modelsimportbuild_modelfromGroundingDINO.groundingdino.utilimportbox_opsfromGroundingDINO.groundingdino.util.slconfigimportSLConfigfromGroundingDINO.groundingdino.util.utilsimportclean_state_dict, get_phrases_from_posmapfromGroundingDINO.groundingdino.util.inferenceimportannotate, load_image, predictimportsupervisionassv# segment anythingfromsegment_anythingimportbuild_sam, SamPredictorimportcv2importnumpyasnpimportmatplotlib.pyplotasplt# diffusersimportPILimportrequestsimporttorchfromioimportBytesIOfromdiffusersimportStableDiffusionInpaintPipelinefromhuggingface_hubimporthf_hub_download

然后我们设置处理的设备：

 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

然后我们创建一个 GroundingDino 模型的实例。

 defload_model_hf(repo_id, filename, ckpt_config_filename, device='cpu'):cache_config_file=hf_hub_download(repo_id=repo_id, filename=ckpt_config_filename)args=SLConfig.fromfile(cache_config_file) args.device=devicemodel=build_model(args)cache_file=hf_hub_download(repo_id=repo_id, filename=filename)checkpoint=torch.load(cache_file, map_location=device)log=model.load_state_dict(clean_state_dict(checkpoint['model']), strict=False)print("Model loaded from {} \n => {}".format(cache_file, log))_=model.eval()returnmodel   ckpt_repo_id="ShilongLiu/GroundingDINO"ckpt_filenmae="groundingdino_swinb_cogcoor.pth"ckpt_config_filename="GroundingDINO_SwinB.cfg.py"groundingdino_model=load_model_hf(ckpt_repo_id, ckpt_filenmae, ckpt_config_filename, device)

下面开始创建SAM 模型，定义模型并创建一个实例。

 ! wget https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pthsam_checkpoint ='sam_vit_h_4b8939.pth'sam_predictor = SamPredictor(build_sam(checkpoint=sam_checkpoint).to(device))

这里我们使用与训练的 vit_h 模型，下面就是扩散模型了：

 sd_pipe=StableDiffusionInpaintPipeline.from_pretrained("stabilityai/stable-diffusion-2-inpainting",torch_dtype=torch.float16,).to(device)

然后我们开始测试：

 # Load image defdownload_image(url, image_file_path):r=requests.get(url, timeout=4.0)ifr.status_code!=requests.codes.ok:assertFalse, 'Status code error: {}.'.format(r.status_code)withImage.open(BytesIO(r.content)) asim:im.save(image_file_path)print('Image downloaded from url: {} and saved to: {}.'.format(url, image_file_path))local_image_path="assets/inpaint_demo.jpg"image_url="https://images.rawpixel.com/image_800/cHJpdmF0ZS9sci9pbWFnZXMvd2Vic2l0ZS8yMDIyLTA1L3Vwd2s2MTc3Nzk0MS13aWtpbWVkaWEtaW1hZ2Uta293YnN1MHYuanBn.jpg"download_image(image_url, local_image_path)image_source, image=load_image(local_image_path)Image.fromarray(image_source)

先使用Grounding Dino 进行检测：

 # detect object using grounding DINOdefdetect(image, text_prompt, model, box_threshold=0.3, text_threshold=0.25):boxes, logits, phrases=predict(model=model, image=image, caption=text_prompt,box_threshold=box_threshold,text_threshold=text_threshold)annotated_frame=annotate(image_source=image_source, boxes=boxes, logits=logits, phrases=phrases)annotated_frame=annotated_frame[...,::-1] # BGR to RGB returnannotated_frame, boxesannotated_frame, detected_boxes=detect(image, text_prompt="bench", model=groundingdino_model)Image.fromarray(annotated_frame)

让我们看看结果：

然后使用 SAM 分割这个狐狸：

 defsegment(image, sam_model, boxes):sam_model.set_image(image)H, W, _=image.shapeboxes_xyxy=box_ops.box_cxcywh_to_xyxy(boxes) *torch.Tensor([W, H, W, H])transformed_boxes=sam_model.transform.apply_boxes_torch(boxes_xyxy.to(device), image.shape[:2])masks, _, _=sam_model.predict_torch(point_coords=None,point_labels=None,boxes=transformed_boxes,multimask_output=False,)returnmasks.cpu()defdraw_mask(mask, image, random_color=True):ifrandom_color:color=np.concatenate([np.random.random(3), np.array([0.8])], axis=0)else:color=np.array([30/255, 144/255, 255/255, 0.6])h, w=mask.shape[-2:]mask_image=mask.reshape(h, w, 1) *color.reshape(1, 1, -1)annotated_frame_pil=Image.fromarray(image).convert("RGBA")mask_image_pil=Image.fromarray((mask_image.cpu().numpy() *255).astype(np.uint8)).convert("RGBA")returnnp.array(Image.alpha_composite(annotated_frame_pil, mask_image_pil))segmented_frame_masks=segment(image_source, sam_predictor, boxes=detected_boxes)annotated_frame_with_mask=draw_mask(segmented_frame_masks[0][0], annotated_frame)Image.fromarray(annotated_frame_with_mask)

这样就可以通过上面的分割结果为的扩散模型生成掩码：

 # create mask images mask=segmented_frame_masks[0][0].cpu().numpy()inverted_mask= ((1-mask) *255).astype(np.uint8)image_source_pil=Image.fromarray(image_source)image_mask_pil=Image.fromarray(mask)inverted_image_mask_pil=Image.fromarray(inverted_mask)display(*[image_source_pil, image_mask_pil, inverted_image_mask_pil])

绘时我们还需要一个背景的掩码，这个就是上面掩码的反操作

 defgenerate_image(image, mask, prompt, negative_prompt, pipe, seed):# resize for inpainting w, h=image.sizein_image=image.resize((512, 512))in_mask=mask.resize((512, 512))generator=torch.Generator(device).manual_seed(seed) result=pipe(image=in_image, mask_image=in_mask, prompt=prompt, negative_prompt=negative_prompt, generator=generator)result=result.images[0]returnresult.resize((w, h))

然后我们可以开始改图，输入一个提示：

 prompt=" a brown bulldog"negative_prompt="low resolution, ugly"seed=-1# for reproducibility generated_image=generate_image(image=image_source_pil, mask=image_mask_pil, prompt=prompt, negative_prompt=negative_prompt, pipe=sd_pipe, seed=seed)generated_image

或者用上面的背景掩码来修改背景：

 prompt="a hill with grasses ,weak sunlight "negative_prompt="people, low resolution, ugly"seed=32# for reproducibility generated_image=generate_image(image_source_pil, inverted_image_mask_pil, prompt, negative_prompt, sd_pipe, seed)generated_image