included date and adjusted the schedule

deploy.py

@@ -1,5 +1,5 @@
 from authenticate import client
-from variables import FUNCTION_NAME, FUNCTION_EXTERNAL_ID
+from variables import FUNCTION_NAME, FUNCTION_DESCRIPTION, FUNCTION_EXTERNAL_ID, FUNCTION_FOLDER, FUNCTION_RUNTIME, FUNCTION_SCHEDULE_NAME, FUNCTION_SCHEDULE
 
 
 if __name__ == "__main__":
@@ -12,15 +12,15 @@
     func = client.functions.create(
         name=FUNCTION_NAME,
         external_id=FUNCTION_EXTERNAL_ID,
-        folder=".",
-        description="Scrape finn realestate data",
-        runtime="py311",
+        folder=FUNCTION_FOLDER,
+        description=FUNCTION_DESCRIPTION,
+        runtime=FUNCTION_RUNTIME,
         )
     print(f"Created function with external_id: {FUNCTION_EXTERNAL_ID}")
 
     schedule = client.functions.schedules.create(
-        name="run-every-hour",
-        cron_expression="0 * * * *",
+        name=FUNCTION_SCHEDULE_NAME,
+        cron_expression=FUNCTION_SCHEDULE,
         function_id=func.id,
         )
     print(f"Created schedule for function with external_id: {FUNCTION_EXTERNAL_ID}")

handler.py

@@ -1,6 +1,7 @@
 import asyncio
 import pandas as pd
 import time
+from datetime import datetime
 from cognite.client import CogniteClient
 from scraper import scraper
 from variables import DATABASE_NAME, TABLE_NAME, ID_COLUMN_NAME
@@ -28,6 +29,7 @@ def main(client, nr_of_pages, price_from, price_to):
         df = pd.DataFrame(scraped_data).fillna('NaN')
         if ID_COLUMN_NAME in df.columns:
             df = df.set_index(ID_COLUMN_NAME)
+            df["Date"] = str(datetime.now().date())
             client.raw.rows.insert_dataframe(DATABASE_NAME, TABLE_NAME, df)
         else:
             print(f"Column '{ID_COLUMN_NAME}' not found in scraped data.")

variables.py

@@ -16,6 +16,11 @@
 
 FUNCTION_NAME = "scrape_finn_realestate_data"
 FUNCTION_EXTERNAL_ID = "fn_001_scrape_finn_realestate_data"
+FUNCTION_DESCRIPTION = "Scrape finn realestate data"
+FUNCTION_FOLDER = "."
+FUNCTION_RUNTIME = "py311"
+FUNCTION_SCHEDULE_NAME = "run-every-hour"
+FUNCTION_SCHEDULE = "0 */2 * * *"
 
 PRICE_MIN_MIN = 0
 PRICE_MIN_MAX = 20000000