Freeze time

cdb_rest/models.py

@@ -93,6 +93,7 @@ class PayloadList(models.Model):
     description = models.CharField(max_length=255, db_column='description', null=True)
     global_tag = models.ForeignKey(GlobalTag, related_name='payload_lists', on_delete=models.CASCADE, null=True)
     payload_type = models.ForeignKey(PayloadType, on_delete=models.CASCADE)
+    #freeze_time = models.BigIntegerField(db_column='freeze_time', null=True)
     created = models.DateTimeField(auto_now_add=True, db_column='created')
     updated = models.DateTimeField(auto_now=True, db_column='updated')
 
@@ -111,9 +112,11 @@ class PayloadIOV(models.Model):
     payload_url = models.CharField(max_length=255, db_column='payload_url')
     major_iov = models.BigIntegerField(db_column='major_iov')
     minor_iov = models.BigIntegerField(db_column='minor_iov')
-    payload_list = models.ForeignKey(PayloadList, related_name='payload_iov', on_delete=models.CASCADE)
+    major_iov_end = models.BigIntegerField(db_column='major_iov_end')
+    minor_iov_end = models.BigIntegerField(db_column='minor_iov_end')
+    payload_list = models.ForeignKey(PayloadList, related_name='payload_iov', on_delete=models.CASCADE, null=True)
     description = models.CharField(max_length=255, db_column='description', null=True)
-    created = models.DateTimeField(auto_now_add=True, db_column='created')
+    inserted = models.DateTimeField(auto_now_add=True, db_column='created')
     updated = models.DateTimeField(auto_now=True, db_column='updated')
 
     class Meta:

cdb_rest/serializers.py

@@ -40,7 +40,7 @@ class PayloadListSerializer(serializers.ModelSerializer):
     #                                        queryset=GlobalTag.objects.all())
     class Meta:
         model = PayloadList
-        fields = ("id", "name", "global_tag", "payload_type", "created")
+        fields = ("id", "name", "global_tag", "payload_type", "freeze_time", "created")
         #depth = 1
 
 
@@ -50,7 +50,7 @@ class PayloadIOVSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = PayloadIOV
-        fields = ("id", "payload_url", "major_iov", "minor_iov", "payload_list", "created")
+        fields = ("id", "payload_url", "major_iov", "minor_iov", "major_iov_end", "minor_iov_end", "payload_list", "inserted")
         #depth = 1
 
 class PayloadListReadSerializer(serializers.ModelSerializer):

cdb_rest/urls.py

@@ -2,7 +2,7 @@
 from cdb_rest.views import GlobalTagListCreationAPIView, GlobalTagDetailAPIView, GlobalTagStatusCreationAPIView, GlobalTagTypeCreationAPIView
 from cdb_rest.views import PayloadListListCreationAPIView, PayloadTypeListCreationAPIView, PayloadIOVListCreationAPIView, PayloadListDetailAPIView
 from cdb_rest.views import PayloadIOVsListAPIView, PayloadIOVsList2APIView, PayloadIOVsRangesListAPIView, PayloadListDetailAPIView, PayloadIOVDetailAPIView
-from cdb_rest.views import PayloadListAttachAPIView, GlobalTagChangeStatusAPIView
+from cdb_rest.views import PayloadListAttachAPIView, GlobalTagChangeStatusAPIView, PayloadIOVAttachAPIView
 from cdb_rest.views import PayloadIOVBulkCreationAPIView
 
 #from cdb_rest.views import GlobalTagCreateAPIView
@@ -31,6 +31,7 @@
     path('bulk_piov', PayloadIOVBulkCreationAPIView.as_view(), name="bulk_payload_iov"),
 
     path('pl_attach', PayloadListAttachAPIView.as_view(), name="payload_list_attach"),
+    path('piov_attach', PayloadIOVAttachAPIView.as_view(), name="payload_iov_list_attach"),
 
 
     #get GT PayloadIOVs

cdb_rest/views.py

@@ -1,3 +1,5 @@
+import sys
+
 from rest_framework.generics import ListCreateAPIView, RetrieveUpdateDestroyAPIView, CreateAPIView, ListAPIView, UpdateAPIView, RetrieveAPIView
 from rest_framework.response import Response
 from rest_framework import status
@@ -203,31 +205,34 @@ def list(self, request):
 
     def create(self, request, *args, **kwargs):
         data = request.data
-        #pList = PayloadList.objects.values_list('id', flat=True).get(name=data['payload_list'])
-        pList = PayloadList.objects.get(name=data['payload_list'])
-        data['payload_list'] = pList.id
 
-        #Check if PL is attached and unlocked
-        if pList.global_tag:
-            if pList.global_tag.status_id == 'locked':
-                return Response({"detail": "Global Tag is locked."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-
-        #Check if  timestamp is greater than the last one
-        piovs = PayloadIOV.objects.filter(payload_list = pList)
-        #print(piovs.order_by('payload_list_id','-major_iov','-minor_iov').distinct('payload_list_id').values_list('id',flat=True))
-        if piovs:
-            max_maj_iov, max_min_iov = piovs.order_by('-major_iov', '-minor_iov').values_list('major_iov','minor_iov')[0]
-            if (data['major_iov'] < max_maj_iov) or ((data['major_iov'] == max_maj_iov) and (data['minor_iov'] <= max_min_iov)):
-                err_msg = "%s PayloadIOV should be greater than: %d %d. Provided IOV: %d %d" % \
-                          (data['payload_url'], max_maj_iov, max_min_iov, data['major_iov'], data['minor_iov'])
-                return Response({"detail": err_msg}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+        if (data['major_iov_end'] == None or data['minor_iov_end'] == None):
+            data['major_iov_end'] == sys.maxsize
+            data['minor_iov_end'] == sys.maxsize
+
+#        if (data['major_iov_end'] == None):
+#            data['minor_iov_end'] == None
+#        elif ((data['minor_iov'] == None) and (data['major_iov_end'] < data['major_iov'])):
+#            data['minor_iov_end'] == None
+#            err_msg = "%s PayloadIOV ending IOVs should be greater or equel than starting. Provided end IOVs: major_iov: %d major_iov_end: %d" % \
+#                      (data['payload_url'], data['major_iov'], data['major_iov_end'])
+#            return Response({"detail": err_msg}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+#        elif((data['major_iov_end'] < data['major_iov']) or ((data['major_iov_end'] == data['major_iov']) and (data['minor_iov_end'] < data['minor_iov']))):
+        if ((data['major_iov_end'] < data['major_iov']) or (
+                (data['major_iov_end'] == data['major_iov']) and (data['minor_iov_end'] < data['minor_iov']))):
+            err_msg = "%s PayloadIOV ending IOVs should be greater or equel than starting. Provided end IOVs: major_iov: %d major_iov_end: %d minor_iov: %d minor_iov_end: %d" % \
+                      (data['payload_url'], data['major_iov'], data['major_iov_end'], data['minor_iov'], data['minor_iov_end'])
+            return Response({"detail": err_msg}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+        else:
+            pass
+
 
         serializer = self.get_serializer(data=data)
         serializer.is_valid(raise_exception=True)
         self.perform_create(serializer)
-        pList.save()
+        #pList.save()
         ret = serializer.data
-        ret['payload_list'] = pList.name
+        #ret['payload_list'] = pList.name
         return Response(ret)
 
 class PayloadIOVDetailAPIView(RetrieveAPIView):
@@ -455,6 +460,106 @@ def put(self, request, *args, **kwargs):
         #serializer.data['global_tag'] = gTag.name
         return Response(ret)
 
+
+class PayloadIOVAttachAPIView(UpdateAPIView):
+
+    serializer_class = PayloadIOVSerializer
+
+    @transaction.atomic
+    def put(self, request, *args, **kwargs):
+
+        data = request.data
+
+        try:
+            pList = PayloadList.objects.get(name=data['payload_list'])
+        except:
+            return Response({"detail": "PayloadList not found."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+        try:
+            piov = PayloadIOV.objects.get(id=data['piov_id'])
+        except:
+            return Response({"detail": "PayloadIOV not found."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+        is_gt_locked = False
+        # Check if PL is attached and unlocked
+        if pList.global_tag:
+            if pList.global_tag.status_id == 'locked':
+                is_gt_locked = True
+
+
+        list_piovs = PayloadIOV.objects.filter(payload_list=pList)
+
+        #Check if new PayloadIOV overlaps
+        if(is_gt_locked):
+
+            # Check if Payload with same start IOVs already attached
+            piovs = list_piovs.filter(major_iov=piov.major_iov, minor_iov=piov.minor_iov)
+            if (piovs):
+                payload_url, major_iov, minor_iov = piovs.values_list('payload_url', 'major_iov', 'minor_iov')[0]
+                err_msg = "PayloadIOV with starting IOVs %d %d already attached: %s" % \
+                          (major_iov, minor_iov, payload_url)
+                return Response({"detail": err_msg}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+            # Special case for Online GT - allow open IOV recover last open IOV
+            special_case = False
+            if(piov.major_iov == sys.maxsize and piov.minor_iov == sys.maxsize):
+                piovs = list_piovs.all().order_by('-major_iov', '-minor_iov')
+                if(piovs):
+                    major_iov_end, minor_iov_end = piovs.values_list('major_iov_end','minor_iov_end')[0]
+                    if (major_iov_end == sys.maxsize and minor_iov_end == sys.maxsize):
+                        special_case = True
+
+            if not special_case:
+                piovs = list_piovs.filter( Q(major_iov__lt = piov.major_iov) | Q(major_iov = piov.major_iov, minor_iov__lt = piov.minor_iov) ).order_by('-major_iov', '-minor_iov')
+                if(piovs):
+                    major_iov_end, minor_iov_end = piovs.values_list('major_iov_end','minor_iov_end')[0]
+                    if (piov.major_iov < major_iov_end) or ((piov.major_iov == major_iov_end) and (piov.minor_iov < minor_iov_end)):
+                        err_msg = "%s PayloadIOV starting IOVs should be equal or greater than: %d %d. Provided start IOVs: %d %d" % \
+                                              (piov.payload_url, major_iov_end, minor_iov_end, piov.major_iov, piov.minor_iov)
+                        return Response({"detail": err_msg}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+                piovs = list_piovs.filter( Q(major_iov__gt = piov.major_iov) | Q(major_iov = piov.major_iov, minor_iov__gt = piov.minor_iov) ).order_by('major_iov', 'minor_iov')
+                if (piovs):
+                    major_iov, minor_iov = piovs.values_list('major_iov', 'minor_iov')[0]
+                    if (piov.major_iov_end > major_iov) or ((piov.major_iov_end == major_iov) and (piov.minor_iov_end > minor_iov)):
+                        err_msg = "%s PayloadIOV ending IOVs should be equal or less than: %d %d. Provided end IOVs: %d %d" % \
+                              (piov.payload_url, major_iov, minor_iov, piov.major_iov_end, piov.minor_iov_end)
+                        return Response({"detail": err_msg}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+        else:
+
+            #piovs fully recovered with inserted to be detached
+            piovs = list_piovs.filter(
+                Q(major_iov__gt=piov.major_iov) | Q(major_iov=piov.major_iov, minor_iov__gte=piov.minor_iov)).filter(
+                Q(major_iov_end__lt=piov.major_iov_end) | Q(major_iov_end=piov.major_iov_end, minor_iov_end__lte=piov.minor_iov_end)).update(payload_list=None)
+
+            #cut the end iovs of the previous piov
+            piovs = list_piovs.filter( Q(major_iov__lt = piov.major_iov) | Q(major_iov = piov.major_iov, minor_iov__lte = piov.minor_iov) ).order_by('-major_iov', '-minor_iov')
+            if(piovs):
+                major_iov_end, minor_iov_end = piovs.values_list('major_iov_end','minor_iov_end')[0]
+                if (piov.major_iov < major_iov_end) or ((piov.major_iov == major_iov_end) and ((piov.minor_iov < minor_iov_end) or (minor_iov_end == None))):
+                    piovs[0].update(major_iov_end = piov.major_iov, minor_iov_end = piov.minor_iov )
+
+            # cut the starting iovs of the next piov
+            piovs = list_piovs.filter( Q(major_iov__gt = piov.major_iov) | Q(major_iov = piov.major_iov, minor_iov__gt = piov.minor_iov) ).order_by('major_iov', 'minor_iov')
+            if (piovs):
+                major_iov, minor_iov = piovs.values_list('major_iov', 'minor_iov')[0]
+                if (piov.major_iov_end == None) or (piov.major_iov_end > major_iov) or ((piov.major_iov_end == major_iov) and ((piov.minor_iov_end > minor_iov) or (piov.minor_iov_end == None))):
+                    piovs[0].update(major_iov=piov.major_iov_end, minor_iov=piov.minor_iov_end)
+
+        piov.payload_list = pList
+
+        self.perform_update(piov)
+
+        #Update time for the pL
+        self.perform_update(pList)
+
+        serializer = PayloadIOVSerializer(piov)
+        #print(serializer.data['global_tag'])
+        #json = JSONRenderer().render(serializer.data)
+        ret = serializer.data
+
+        return Response(ret)
+
 class GlobalTagChangeStatusAPIView(UpdateAPIView):
 
     serializer_class = GlobalTagCreateSerializer

nopayloaddb/settings.py

@@ -90,7 +90,6 @@
     'default': {
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
 
-
         'NAME':     os.environ.get("POSTGRES_DB",       default='dbname'),
         'USER':     os.environ.get("POSTGRES_USER",     default='login'),
         'PASSWORD': os.environ.get("POSTGRES_PASSWORD", default='password'),

requirements.txt

@@ -11,7 +11,4 @@ sqlparse==0.4.2
 typing-extensions==3.10.0.2
 requests==2.26.0
 urllib3==1.26.7
-charset-normalizer==2.0.7
-
-
-
+charset-normalizer==2.0.7