diff --git a/boot/bootutil/src/loader.c b/boot/bootutil/src/loader.c
index 9c14c03e6..151944f1b 100644
--- a/boot/bootutil/src/loader.c
+++ b/boot/bootutil/src/loader.c
@@ -70,6 +70,9 @@ int pcd_version_cmp_net(const struct flash_area *fap, struct image_header *hdr);
 BOOT_LOG_MODULE_DECLARE(mcuboot);
 
 static struct boot_loader_state boot_data;
+#ifdef PM_S1_ADDRESS
+static bool owner_nsib[BOOT_IMAGE_NUMBER] = {false};
+#endif
 
 #if (BOOT_IMAGE_NUMBER > 1)
 #define IMAGES_ITER(x) for ((x) = 0; (x) < BOOT_IMAGE_NUMBER; ++(x))
@@ -1286,6 +1289,9 @@ boot_validated_swap_type(struct boot_loader_state *state,
     int swap_type;
     FIH_DECLARE(fih_rc, FIH_FAILURE);
     bool upgrade_valid = false;
+#if defined(PM_S1_ADDRESS)
+    owner_nsib[BOOT_CURR_IMG(state)] = false;
+#endif
 
 #if defined(PM_S1_ADDRESS) || defined(CONFIG_SOC_NRF5340_CPUAPP)
     const struct flash_area *secondary_fa =
@@ -1342,6 +1348,7 @@ boot_validated_swap_type(struct boot_loader_state *state,
                         && reset_addr < (nsib_fa->fa_off + nsib_fa->fa_size)) {
                         /* Set primary to be NSIB upgrade slot */
                         BOOT_IMG_AREA(state, 0) = nsib_fa;
+                        owner_nsib[BOOT_CURR_IMG(state)] = true;
                     }
 #else
                 return BOOT_SWAP_TYPE_NONE;
@@ -1352,6 +1359,10 @@ boot_validated_swap_type(struct boot_loader_state *state,
                 /* The image in the secondary slot is not intended for any */
                 return BOOT_SWAP_TYPE_NONE;
             }
+
+            if ((primary_fa->fa_off == PM_S0_ADDRESS) || (primary_fa->fa_off == PM_S1_ADDRESS)) {
+                owner_nsib[BOOT_CURR_IMG(state)] = true;
+            }
         }
 #endif /* PM_S1_ADDRESS */
         sec_slot_mark_assigned(state);
@@ -2291,6 +2302,13 @@ check_downgrade_prevention(struct boot_loader_state *state)
     uint32_t security_counter[2];
     int rc;
 
+#if defined(PM_S1_ADDRESS)
+    if (owner_nsib[BOOT_CURR_IMG(state)]) {
+        /* Downgrade prevention on S0/S1 image is managed by NSIB */
+        return 0;
+    }
+#endif
+
     if (MCUBOOT_DOWNGRADE_PREVENTION_SECURITY_COUNTER) {
         /* If there was security no counter in slot 0, allow swap */
         rc = bootutil_get_img_security_cnt(&(BOOT_IMG(state, 0).hdr),