Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

reuse github workflow from superdesk/superdesk #644

Merged
merged 5 commits into from
Nov 21, 2024
Merged

reuse github workflow from superdesk/superdesk #644

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
209e3fa
use github actions from superdesk/superdesk
petrjasek Nov 20, 2024
ec7c4ac
fix black
petrjasek Nov 20, 2024
28632cb
run pytest on github actions
petrjasek Nov 21, 2024
24d41c3
set versions in dev-requirements
petrjasek Nov 21, 2024
1d0885c
fix dev-requirements
petrjasek Nov 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 8 additions & 61 deletions .github/workflows/tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,69 +1,16 @@
name: "CI"

on:
[push, pull_request]
on: [push, pull_request]

jobs:

server:
runs-on: ubuntu-latest

defaults:
run:
working-directory: server

services:
redis:
image: redis:alpine
ports:
- "6379:6379"

mongo:
image: mongo:4
ports:
- "27017:27017"

elastic:
image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
ports:
- "9200:9200"
env:
discovery.type: single-node

steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v3
with:
python-version: 3.8
cache: 'pip'
cache-dependency-path: server/requirements.txt

- name: apt-get
run: |
sudo apt-get update
sudo apt-get -y install libxml2-dev libxmlsec1-dev libxmlsec1-openssl libexempi-dev

- name: pip install
run: |
python -m pip install --upgrade pip wheel setuptools
pip install -r dev-requirements.txt

- run: flake8
- run: pytest --log-level=ERROR --disable-warnings
- run: mypy .
- run: python manage.py app:initialize_data
strategy:
matrix:
python-version: ['3.8', '3.10']
uses: superdesk/superdesk/.github/workflows/server.yml@develop
with:
python-version: ${{ matrix.python-version }}

client:
runs-on: ubuntu-latest

defaults:
run:
working-directory: client

steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 12.x
- run: npm ci || npm install
- run: npm run build
uses: superdesk/superdesk/.github/workflows/client.yml@develop
8 changes: 4 additions & 4 deletions server/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def get_app(config=None):
if config is None:
config = {}

config['APP_ABSPATH'] = os.path.abspath(os.path.dirname(__file__))
config["APP_ABSPATH"] = os.path.abspath(os.path.dirname(__file__))

for key in dir(settings):
if key.isupper():
Expand All @@ -34,9 +34,9 @@ def get_app(config=None):
return app


if __name__ == '__main__':
if __name__ == "__main__":
debug = True
host = '0.0.0.0'
port = int(os.environ.get('PORT', '5000'))
host = "0.0.0.0"
port = int(os.environ.get("PORT", "5000"))
app = get_app()
app.run(host=host, port=port, debug=debug, use_reloader=debug)
17 changes: 11 additions & 6 deletions server/belga/ai_proxy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,16 @@
def init_app(app: Flask) -> None:
belga_ai_url = app.config.get("BELGA_AI_URL")
if belga_ai_url is None:
logger.warning("'BELGA_AI_URL' config not set, HTTP Proxy will not be available")
logger.warning(
"'BELGA_AI_URL' config not set, HTTP Proxy will not be available"
)
return

register_http_proxy(app, HTTPProxy(
endpoint_name="belga.ai_proxy",
internal_url="belga/ai",
external_url=belga_ai_url,
))
register_http_proxy(
app,
HTTPProxy(
endpoint_name="belga.ai_proxy",
internal_url="belga/ai",
external_url=belga_ai_url,
),
)
233 changes: 141 additions & 92 deletions server/belga/command/contacts_import.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,121 +14,169 @@ def import_contacts_via_json_file(path_file):
:param unit_test:
:return:
"""
with open(path_file, 'rt', encoding='utf-8') as contacts_data:
with open(path_file, "rt", encoding="utf-8") as contacts_data:
json_data = json.load(contacts_data)
contact_service = get_resource_service('contacts')
contact_service = get_resource_service("contacts")
docs = []
count_items = len(json_data)
count_import = 0
for item in json_data:

# Validate data contacts
# contact is not name and email, not import
if not item.get('email') and not item.get('personalEmail') and not item.get('firstName') and not item.get(
'lastName') and not item.get('company'):
logger.info("contact (id:%s) is not name, company and email, not import." % str(item.get('contactId')))
if (
not item.get("email")
and not item.get("personalEmail")
and not item.get("firstName")
and not item.get("lastName")
and not item.get("company")
):
logger.info(
"contact (id:%s) is not name, company and email, not import."
% str(item.get("contactId"))
)
continue
else:
# contact same name, email, phone, not import
query = {'first_name': item.get('firstName', ''), "last_name": item.get('lastName', '')}
query = {
"first_name": item.get("firstName", ""),
"last_name": item.get("lastName", ""),
}
# if first and last name is empty, check company
if not item.get('firstName') and not item.get('lastName'):
query = ({'organisation': item.get('company', '')})
if not item.get("firstName") and not item.get("lastName"):
query = {"organisation": item.get("company", "")}
and_query = []
if item.get('email'):
and_query.append({"contact_email": item.get('email')})
if item.get('personalEmail'):
and_query.append({"contact_email": item.get('personalEmail')})
if item.get('directPhone1'):
and_query.append({"contact_phone.number": item.get('directPhone1')})
if item.get('directPhone2'):
and_query.append({"contact_phone.number": item.get('directPhone2')})
if item.get('phoneGeneral'):
and_query.append({"contact_phone.number": item.get('phoneGeneral')})
if item.get('personalPhone'):
and_query.append({"contact_phone.number": item.get('personalPhone')})
if item.get("email"):
and_query.append({"contact_email": item.get("email")})
if item.get("personalEmail"):
and_query.append({"contact_email": item.get("personalEmail")})
if item.get("directPhone1"):
and_query.append({"contact_phone.number": item.get("directPhone1")})
if item.get("directPhone2"):
and_query.append({"contact_phone.number": item.get("directPhone2")})
if item.get("phoneGeneral"):
and_query.append({"contact_phone.number": item.get("phoneGeneral")})
if item.get("personalPhone"):
and_query.append(
{"contact_phone.number": item.get("personalPhone")}
)
if and_query:
query.update({"$and": and_query})

contacts = contact_service.find(query)

if len(list(contacts)):
logger.info(
"contact (id:%s) is exist, same name(%s %s), email(%s, %s), phone(%s, %s), not import" % (
str(item.get('contactId')), item.get('firstName'), item.get('lastName'),
item.get('email', ''), item.get('personalEmail', ''), item.get('directPhone1', ''),
item.get('directPhone2', '')))
"contact (id:%s) is exist, same name(%s %s), email(%s, %s), phone(%s, %s), not import"
% (
str(item.get("contactId")),
item.get("firstName"),
item.get("lastName"),
item.get("email", ""),
item.get("personalEmail", ""),
item.get("directPhone1", ""),
item.get("directPhone2", ""),
)
)
continue
doc = {}
logger.info("contact (id:%s) is insert successfully:" % str(item.get('contactId')))
logger.info(
"contact (id:%s) is insert successfully:" % str(item.get("contactId"))
)
# mapping data
doc.setdefault('schema', {}).update({"is_active": True,
"public": item.get("belgaPublic", True)
})
doc['organisation'] = item.get("company", "")
doc['first_name'] = item.get("firstName", "")
doc['last_name'] = item.get("lastName", "")
if item.get('function'):
doc['job_title'] = item.get("function")
if item.get('mobile247'):
doc.setdefault('mobile', []).append({'number': item.get('mobile247'),
'usage': 'Business',
'public': True
})
if item.get('personalMobile'):
doc.setdefault('mobile', []).append({'number': item.get('personalMobile'),
'usage': 'Confidential',
'public': True
})
if item.get('phoneGeneral'):
doc.setdefault('contact_phone', []).append({'number': item.get('phoneGeneral'),
'usage': 'Business',
'public': True
})
if item.get('directPhone1'):
doc.setdefault('contact_phone', []).append({'number': item.get('directPhone1'),
'usage': 'Business',
'public': True
})
if item.get('directPhone2'):
doc.setdefault('contact_phone', []).append({'number': item.get('directPhone2'),
'usage': 'Business',
'public': True
})
if item.get('personalPhone'):
doc.setdefault('contact_phone', []).append({'number': item.get('personalPhone'),
'usage': 'Confidential',
'public': True
})
doc['fax'] = ''
if item.get('email'):
doc.setdefault('contact_email', []).append(item.get('email'))
if item.get('personalEmail'):
doc.setdefault('contact_email', []).append(item.get('personalEmail'))
if item.get('twitter'):
doc['twitter'] = item.get('twitter')
if item.get('personalTwitter'):
doc['twitter_personal'] = item.get('personalTwitter')
if item.get('facebook'):
doc['facebook'] = item.get('facebook')
if item.get('personalFacebook'):
doc['facebook_personal'] = item.get('personalFacebook')
if item.get('url'):
doc['website'] = item.get('url')
if item.get('professionalAddress'):
doc.setdefault('contact_address', []).append(item.get('professionalAddress'))
if item.get('professionalAddress'):
doc.setdefault('contact_address', []).append(item.get('personalAddress'))
if item.get('Comment1'):
doc['notes'] = item.get('Comment1', '')
if item.get('keywords'):
doc['keywords'] = item.get('keywords')
doc.setdefault("schema", {}).update(
{"is_active": True, "public": item.get("belgaPublic", True)}
)
doc["organisation"] = item.get("company", "")
doc["first_name"] = item.get("firstName", "")
doc["last_name"] = item.get("lastName", "")
if item.get("function"):
doc["job_title"] = item.get("function")
if item.get("mobile247"):
doc.setdefault("mobile", []).append(
{
"number": item.get("mobile247"),
"usage": "Business",
"public": True,
}
)
if item.get("personalMobile"):
doc.setdefault("mobile", []).append(
{
"number": item.get("personalMobile"),
"usage": "Confidential",
"public": True,
}
)
if item.get("phoneGeneral"):
doc.setdefault("contact_phone", []).append(
{
"number": item.get("phoneGeneral"),
"usage": "Business",
"public": True,
}
)
if item.get("directPhone1"):
doc.setdefault("contact_phone", []).append(
{
"number": item.get("directPhone1"),
"usage": "Business",
"public": True,
}
)
if item.get("directPhone2"):
doc.setdefault("contact_phone", []).append(
{
"number": item.get("directPhone2"),
"usage": "Business",
"public": True,
}
)
if item.get("personalPhone"):
doc.setdefault("contact_phone", []).append(
{
"number": item.get("personalPhone"),
"usage": "Confidential",
"public": True,
}
)
doc["fax"] = ""
if item.get("email"):
doc.setdefault("contact_email", []).append(item.get("email"))
if item.get("personalEmail"):
doc.setdefault("contact_email", []).append(item.get("personalEmail"))
if item.get("twitter"):
doc["twitter"] = item.get("twitter")
if item.get("personalTwitter"):
doc["twitter_personal"] = item.get("personalTwitter")
if item.get("facebook"):
doc["facebook"] = item.get("facebook")
if item.get("personalFacebook"):
doc["facebook_personal"] = item.get("personalFacebook")
if item.get("url"):
doc["website"] = item.get("url")
if item.get("professionalAddress"):
doc.setdefault("contact_address", []).append(
item.get("professionalAddress")
)
if item.get("professionalAddress"):
doc.setdefault("contact_address", []).append(
item.get("personalAddress")
)
if item.get("Comment1"):
doc["notes"] = item.get("Comment1", "")
if item.get("keywords"):
doc["keywords"] = item.get("keywords")
# use original_id check and sync the contact from the belga.
doc['original_id'] = str(item.get('contactId'))
doc["original_id"] = str(item.get("contactId"))
count_import += 1
contact_service.post([doc])
docs.append(doc)
logger.info("number item: " + str(count_items) + ", number imported item: " + str(count_import))
logger.info(
"number item: "
+ str(count_items)
+ ", number imported item: "
+ str(count_import)
)
return docs


Expand All @@ -139,13 +187,14 @@ class ContactImportCommand(superdesk.Command):
"""

option_list = [
superdesk.Option('--file', '-f', dest='contacts_file_path',
default='contacts.json')
superdesk.Option(
"--file", "-f", dest="contacts_file_path", default="contacts.json"
)
]

def run(self, contacts_file_path):
logger.info("import file: " + contacts_file_path)
import_contacts_via_json_file(contacts_file_path)


superdesk.command('contact:import', ContactImportCommand())
superdesk.command("contact:import", ContactImportCommand())
Loading