WIP: python3.10/11, opensearch, invenio upgrades

.github/workflows/e2e-workflow-server.yml

@@ -2,7 +2,7 @@ name: E2E - Server changes with python package installations
 
 on:
   pull_request:
-    branches: [master, ant, faser]
+    branches: [master, next]
     paths:
       - "setup.py"
       - "requirements.txt"

.github/workflows/e2e-workflow-ui.yml

@@ -3,7 +3,7 @@ on:
   push:
     branches: uitest
   pull_request:
-    branches: [master, ant, faser]
+    branches: [master, next]
     paths:
       - "ui/**"
       - ".github/workflows/ui-test.yml"

.github/workflows/e2e-workflow.yml

@@ -5,11 +5,12 @@ on:
       - master
       - qa
       - production
+      - next
   pull_request:
     branches:
       - qa
       - production
-      - faser
+      - next
 jobs:
   Cypress:
     name: Cypress
@@ -25,7 +26,7 @@ jobs:
         run: docker-compose -f docker-compose.e2e.yml run web-api sh scripts/clean-and-init.sh
 
       - name: Update save folder
-        run: docker-compose -f docker-compose.e2e.yml run web-api  cap files location local var/data --default
+        run: docker-compose -f docker-compose.e2e.yml run web-api cap files location local var/data --default
 
       - name: Sleep for 180 seconds
         run: sleep 180s

.github/workflows/lint-commit.yml

@@ -2,7 +2,7 @@ name: Lint Commit
 
 on:
   pull_request:
-    branches: [ master ]
+    branches: [ master, next ]
 
 jobs:
   lint-shellcheck:
@@ -28,7 +28,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: Checking commit quality (messages, signatures, spelling checks, etc)
         env:
@@ -44,7 +44,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: Check Python manifest completeness
         run: |
@@ -59,7 +59,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: Check compliance with Python docstring conventions
         run: |

.github/workflows/lint-push.yml

@@ -2,7 +2,7 @@ name: Lint Source
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, next ]
 
 jobs:
   lint-shellcheck:
@@ -23,7 +23,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: Check compliance with Python docstring conventions
         run: |
@@ -38,7 +38,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: Check compliance with pep8, pyflakes and circular complexity
         run: |
@@ -55,7 +55,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: Run black linter for python
         run: |
@@ -70,7 +70,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: isort your imports, so you don't have to.
         run: |
@@ -100,7 +100,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: Check Python manifest completeness
         run: |
@@ -120,7 +120,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: '3.10'
 
       - name: Checking commit quality (messages, signatures, spelling checks, etc)
         env:

.github/workflows/ui-test.yml

@@ -1,12 +1,12 @@
 name: UI test
 on:
   push:
-    branches: [master, ant]
+    branches: [master, next]
     paths:
       - "ui/**"
       - ".github/workflows/ui-test.yml"
   pull_request:
-    branches: [master, ant, faser]
+    branches: [master, next]
     paths:
       - "ui/**"
       - ".github/workflows/ui-test.yml"

.github/workflows/web-api.yml

@@ -2,13 +2,13 @@ name: Backend tests
 
 on:
   push:
-    branches: [ master, ant ]
+    branches: [master, next]
     paths-ignore:
-    - 'ui/**'
+      - "ui/**"
   pull_request:
-    branches: [ master, ant, faser ]
+    branches: [master, next]
     paths-ignore:
-    - 'ui/**'
+      - "ui/**"
 
 jobs:
   lint-python-changes:
@@ -19,10 +19,11 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: "3.10"
 
       - name: Run darker
         run: |
+          pip install flake8
           pip install darker[isort]
           darker . --check -i -L flake8
 
@@ -40,40 +41,44 @@ jobs:
           POSTGRES_PASSWORD: cap
           POSTGRES_DB: cap
         ports:
-        - 5432:5432
+          - 5432:5432
       rabbitmq:
         image: rabbitmq:3-management
         ports:
           - 5672:5672
-      elasticsearchd:
-        image: docker.elastic.co/elasticsearch/elasticsearch:5.6.4
+      search:
+        image: opensearchproject/opensearch:2.2.0
         env:
+          node.name: search
           bootstrap.memory_lock: true
-          ES_JAVA_OPTS: -Xms1024m -Xmx1024m
+          OPENSEARCH_JAVA_OPTS: -Xms1024m -Xmx1024m
           discovery.type: single-node
-          xpack.security.enabled: false
+          discovery.seed_hosts: search
+          DISABLE_SECURITY_PLUGIN: true
         ports:
           - 9200:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
 
     steps:
       - uses: actions/checkout@v2
 
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.6.9
+          python-version: "3.10"
 
       - name: Install system dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install libsasl2-dev python-dev libldap2-dev libssl-dev libkrb5-dev
+          sudo apt-get install libsasl2-dev python3-dev libldap2-dev libssl-dev libkrb5-dev
 
       - name: Install dependencies
         run: |
-          pip install --upgrade pip==20.2.4
-          pip install -r requirements.txt
-          pip install -e .[all,docs,tests,ldap]
-          pip install -r requirements-local-forks.txt
+          pip install .
 
       - name: Run integration and unit tests
         run: ./run-tests.sh --check-pytest

Dockerfile

@@ -1,20 +1,48 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2018 CERN.
+# Copyright (C) 2023 CERN.
 #
-# CERN Analysis Preservation is free software; you can redistribute it and/or modify it
-# under the terms of the MIT License; see LICENSE file for more details.
-
-FROM gitlab-registry.cern.ch/analysispreservation/base:python3
+# Base cap image to install on with base python 3.10
+
+FROM python:3.10.10
+
+# Certficates configuartion
+ENV PYTHONBUFFERED=0 \
+    SSL_CERT_FILE="/etc/ssl/certs/ca-certificates.crt" \
+    REQUESTS_CA_BUNDLE="/etc/ssl/certs/ca-certificates.crt" \
+    PATH="/root/.local/bin:${PATH}" \
+    POETRY_VIRTUALENVS_CREATE=false
+COPY docker/base/CERN_Root_Certification_Authority_2.pem /usr/local/share/ca-certificates/CERN_Root_Certification_Authority_2.crt
+
+# Install system dependencies
+RUN update-ca-certificates && pip config set global.cert "${REQUESTS_CA_BUNDLE}"
+RUN curl -s -L http://cern.ch/linux/docs/krb5.conf -o /etc/krb5.conf
+RUN curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
+
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    gcc libffi-dev locales libxslt-dev libxml2-dev libssl-dev build-essential python3-dev libldap2-dev libsasl2-dev ldap-utils krb5-user git redis \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set the locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
 
 # Install Invenio
 ENV WORKING_DIR=/opt/cap
 ENV INVENIO_INSTANCE_PATH=${WORKING_DIR}/var/instance
+ENV CAP_FILES_DIR=${WORKING_DIR}/src/var
 
 # Debug off by default
 ARG DEBUG=False
 ENV DEBUG=${DEBUG}
 
+# ENABLE_E2E False by default
+ARG ENABLE_E2E=False
+ENV ENABLE_E2E=${ENABLE_E2E}
+
 # copy everything inside /src
 RUN mkdir -p ${WORKING_DIR}/src
 COPY ./ ${WORKING_DIR}/src
@@ -28,17 +56,13 @@ RUN python -m site --user-site
 
 # Install/create static files
 RUN mkdir -p ${INVENIO_INSTANCE_PATH}
+RUN mkdir -p ${CAP_FILES_DIR}
 
+RUN pip install -e .
 
-RUN pip install --upgrade wheel uwsgi uwsgitop uwsgi-tools
-
-# RUN if [ "$DEBUG" = "True" ]; then pip install -r requirements-devel.txt; fi;
-RUN pip install -r requirements.txt
-RUN pip install -e .[all,xrootd]
-RUN pip install -r requirements-local-forks.txt
-
+RUN cat ./docker/base/CERN_Root_Certification_Authority_2.pem >>  /usr/local/lib/python3.10/site-packages/certifi/cacert.pem
 
-# copy uwsgi config files
+# Copy uwsgi config files
 COPY ./docker/uwsgi/ ${INVENIO_INSTANCE_PATH}
 
 ARG APP_GITHUB_OAUTH_ACCESS_TOKEN
@@ -49,9 +73,10 @@ ENV APP_GITLAB_OAUTH_ACCESS_TOKEN=${APP_GITLAB_OAUTH_ACCESS_TOKEN}
 RUN pip install gunicorn
 
 # Set folder permissions
-RUN chgrp -R 0 ${WORKING_DIR} && \
-    chmod -R g=u ${WORKING_DIR}
+RUN chgrp -R 0 ${WORKING_DIR} ${CAP_FILES_DIR} && \
+    chmod -R g=u ${WORKING_DIR} ${CAP_FILES_DIR}
 
 RUN useradd invenio --uid 1000 --gid 0 && \
-    chown -R invenio:root ${WORKING_DIR}
+    chown -R invenio:root ${WORKING_DIR} ${CAP_FILES_DIR} && \
+    chown -R invenio:root /usr/local/lib/python3.10/site-packages/certifi
 USER 1000

MANIFEST.in

@@ -47,6 +47,7 @@ include .githooks/pre-push
 include .githooks/prepare-commit-msg
 include docker/harbour/backend/Dockerfile
 include docker/harbour/ui/Dockerfile
+include docker/base/Dockerfile
 include docker/haproxy/Dockerfile
 include docker/nginx/Dockerfile
 include docker/postgres/Dockerfile

cap.sample.env

@@ -10,7 +10,7 @@ APP_BROKER_URL=redis://redis:6379/0
 # APP_CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672/
 APP_ACCESS_SESSION_REDIS_HOST=redis
 APP_CELERY_RESULT_BACKEND=redis://redis:6379/2
-APP_SEARCH_ELASTIC_HOSTS=elasticsearch
+APP_SEARCH_HOSTS=opensearch
 APP_JSONSCHEMAS_HOST=localhost:5000
 
 # CERN oAuth client credentials