Skip to content

Commit 05694a9

Browse files
NeonDanielNeonDaniel
authored
Refactor module to remove deprecated code and simplify structure (#33)
* Refactor module to remove deprecated code and simplify structure * Add FastChat config * Update chart versions to resolve deployment errors Update deployments to ensure pod recreation on update
1 parent fb43d63 commit 05694a9

48 files changed

Lines changed: 548 additions & 3803 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/unit_tests.yml‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -19,29 +19,13 @@ jobs:
1919
- name: Install dependencies
2020
run: |
2121
python -m pip install --upgrade pip
22-
pip install -r requirements/requirements.txt
22+
pip install -e .
2323
pip install -r requirements/test_requirements.txt
24-
- name: Run Utils Tests
24+
- name: Run Tests
2525
run: |
26-
pytest tests/test_diana_utils.py --doctest-modules --junitxml=tests/utils-test-results.xml
27-
- name: Upload Utils test results
26+
pytest tests --doctest-modules --junitxml=tests/diana-test-results.xml
27+
- name: Upload test results
2828
uses: actions/upload-artifact@v2
2929
with:
3030
name: utils-test-results
31-
path: tests/utils-test-results.xml
32-
- name: Run Constants Tests
33-
run: |
34-
pytest tests/test_constants.py --doctest-modules --junitxml=tests/constants-test-results.xml
35-
- name: Upload Constants test results
36-
uses: actions/upload-artifact@v2
37-
with:
38-
name: constants-test-results
39-
path: tests/constants-test-results.xml
40-
- name: Run Service Tests
41-
run: |
42-
pytest tests/test_diana_services.py --doctest-modules --junitxml=tests/services-test-results.xml
43-
- name: Upload Services test results
44-
uses: actions/upload-artifact@v2
45-
with:
46-
name: services-test-results
47-
path: tests/services-test-results.xml
31+
path: tests/diana-test-results.xml

‎neon_diana_utils/cli.py‎

Lines changed: 18 additions & 360 deletions
Large diffs are not rendered by default.

‎neon_diana_utils/configuration.py‎

Lines changed: 301 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,301 @@
1+
# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System
2+
# All trademark and other rights reserved by their respective owners
3+
# Copyright 2008-2021 Neongecko.com Inc.
4+
# BSD-3
5+
# Redistribution and use in source and binary forms, with or without
6+
# modification, are permitted provided that the following conditions are met:
7+
# 1. Redistributions of source code must retain the above copyright notice,
8+
# this list of conditions and the following disclaimer.
9+
# 2. Redistributions in binary form must reproduce the above copyright notice,
10+
# this list of conditions and the following disclaimer in the documentation
11+
# and/or other materials provided with the distribution.
12+
# 3. Neither the name of the copyright holder nor the names of its
13+
# contributors may be used to endorse or promote products derived from this
14+
# software without specific prior written permission.
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
17+
# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
22+
# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
23+
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
24+
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25+
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
27+
import click
28+
import yaml
29+
import json
30+
import secrets
31+
import shutil
32+
33+
from pprint import pformat
34+
from typing import Optional
35+
from os import makedirs, listdir, walk, remove
36+
from os.path import expanduser, join, abspath, isfile, isdir, dirname
37+
from ovos_utils.xdg_utils import xdg_config_home
38+
from ovos_utils.log import LOG
39+
40+
41+
def validate_output_path(output_path: str) -> bool:
42+
"""
43+
Ensure the requested output path is available to be written
44+
@returns: True if path is valid, else False
45+
"""
46+
if isfile(output_path):
47+
LOG.warning(f"File already exists: {output_path}")
48+
return False
49+
elif isdir(output_path) and listdir(output_path):
50+
LOG.warning(f"Directory is not empty: {output_path}")
51+
return False
52+
elif not isdir(dirname(output_path)):
53+
makedirs(dirname(output_path))
54+
return True
55+
56+
57+
def make_keys_config(write_config: bool,
58+
output_file: str = None) -> Optional[dict]:
59+
"""
60+
Interactive configuration tool to prompt user for expected API keys and
61+
service accounts to be included in Configuration.
62+
@param write_config: If true, write config to `output_file`
63+
@param output_file: Configuration file to write keys to
64+
@returns: dict configuration
65+
"""
66+
if write_config:
67+
output_file = expanduser(abspath((output_file or join(xdg_config_home(),
68+
"diana",
69+
"diana.yaml"))))
70+
if not validate_output_path(output_file):
71+
click.echo(f"File already exists: {output_file}")
72+
return
73+
74+
api_services = dict()
75+
if click.confirm("Configure API Proxy Services?"):
76+
keys_confirmed = False
77+
while not keys_confirmed:
78+
wolfram_key = click.prompt("Wolfram|Alpha API Key", type=str)
79+
alphavantage_key = click.prompt("AlphaVantage API Key",
80+
type=str)
81+
owm_key = click.prompt("OpenWeatherMap API Key", type=str)
82+
api_services = {
83+
"wolfram_alpha": {"api_key": wolfram_key},
84+
"alpha_vantage": {"api_key": alphavantage_key},
85+
"open_weather_map": {"api_key": owm_key}
86+
}
87+
click.echo(pformat(api_services))
88+
keys_confirmed = click.confirm("Are these keys correct?")
89+
90+
email_config = dict()
91+
if click.confirm("Configure Email Service?"):
92+
config_confirmed = False
93+
while not config_confirmed:
94+
email_addr = click.prompt("Email Address", type=str)
95+
email_password = click.prompt("Password", type=str)
96+
smtp_host = click.prompt("SMTP URL", type=str,
97+
default="smtp.gmail.com")
98+
smtp_port = click.prompt("SMTP Port", type=str,
99+
default="465")
100+
email_config = {"mail": email_addr,
101+
"pass": email_password,
102+
"host": smtp_host,
103+
"port": smtp_port}
104+
click.echo(pformat(email_config))
105+
config_confirmed = \
106+
click.confirm("Is this configuration correct?")
107+
108+
brands_config = dict()
109+
if click.confirm("Configure Brands/Coupons Service?"):
110+
config_confirmed = False
111+
while not config_confirmed:
112+
server_host = click.prompt("SQL Host Address", type=str,
113+
default="trackmybrands.com")
114+
sql_database = click.prompt("SQL Database", type=str,
115+
default="admintr1_drup1")
116+
sql_username = click.prompt("SQL Username", type=str)
117+
sql_password = click.prompt("SQL Password", type=str)
118+
brands_config = {"host": server_host,
119+
"database": sql_database,
120+
"user": sql_username,
121+
"password": sql_password}
122+
click.echo(pformat(brands_config))
123+
config_confirmed = \
124+
click.confirm("Is this configuration correct?")
125+
126+
chatgpt_config = dict()
127+
if click.confirm("Configure ChatGPT Service?"):
128+
config_confirmed = False
129+
while not config_confirmed:
130+
gpt_key = click.prompt("ChatGPT API Key", type=str)
131+
gpt_model = click.prompt("ChatGPT Model", type=str,
132+
default="gpt-3.5-turbo")
133+
gpt_role = click.prompt("ChatGPT Role", type=str,
134+
default="You are trying to give a short "
135+
"answer in less than 40 words.")
136+
gpt_context = click.prompt("ChatGPT Context depth", type=int,
137+
default=3)
138+
max_tokens = click.prompt("Maximum tokens in responses", type=int,
139+
default=100)
140+
chatgpt_config = {
141+
"key": gpt_key,
142+
"model": gpt_model,
143+
"role": gpt_role,
144+
"context_depth": gpt_context,
145+
"max_tokens": max_tokens
146+
}
147+
click.echo(pformat(chatgpt_config))
148+
config_confirmed = \
149+
click.confirm("Is this configuration correct?")
150+
151+
fastchat_config = dict()
152+
if click.confirm("Configure FastChat Service?"):
153+
config_confirmed = False
154+
while not config_confirmed:
155+
model = click.prompt("FastChat Model", type=str,
156+
default="fastchat")
157+
context = click.prompt("FastChat context depth", type=int,
158+
default=3)
159+
max_tokens = click.prompt("Max number of tokens in responses",
160+
type=int, default=128)
161+
num_processes = click.prompt(
162+
"Number of queries to handle in parallel",
163+
type=int, default=1)
164+
num_threads = click.prompt("Number of threads to use per query",
165+
type=int, default=2)
166+
fastchat_config = {
167+
"model": model,
168+
"context_depth": context,
169+
"max_tokens": max_tokens,
170+
"num_parallel_processes": num_processes,
171+
"num_threads_per_process": num_threads
172+
}
173+
click.echo(pformat(fastchat_config))
174+
config_confirmed = click.confirm("Is this configuration correct?")
175+
176+
config = {"keys": {"api_services": api_services,
177+
"emails": email_config,
178+
"track_my_brands": brands_config},
179+
"ChatGPT": chatgpt_config,
180+
"FastChat": fastchat_config
181+
}
182+
if write_config:
183+
click.echo(f"Writing configuration to {output_file}")
184+
with open(output_file, 'w+') as f:
185+
yaml.dump(config, f)
186+
return config
187+
188+
189+
def generate_rmq_config(admin_username: str, admin_password: str,
190+
output_file: str = None) -> dict:
191+
"""
192+
Generate a default configuration for RabbitMQ. This defines all default
193+
users, vhosts, and permissions that may be used with a deployment.
194+
@param admin_username: Username for admin account
195+
@param admin_password: Password for admin account
196+
@param output_file: Optional path to write configuration to
197+
@returns: dict RabbitMQ Configuration
198+
"""
199+
base_config_file = join((dirname(__file__)), "templates",
200+
"rmq_backend_config.yml")
201+
with open(base_config_file) as f:
202+
base_config = yaml.safe_load(f)
203+
for user in base_config['users']:
204+
if user["password"]:
205+
# Skip users with defined passwords
206+
continue
207+
user['password'] = secrets.token_urlsafe(32)
208+
209+
base_config['users'].append({'name': admin_username,
210+
'password': admin_password,
211+
'tags': ['administrator']})
212+
213+
if output_file and validate_output_path(output_file):
214+
with open(output_file, 'w+') as f:
215+
json.dump(base_config, f, indent=2)
216+
return base_config
217+
218+
219+
def generate_mq_auth_config(rmq_config: dict) -> dict:
220+
"""
221+
Generate an MQ auth config from RabbitMQ config
222+
:param rmq_config: RabbitMQ definitions, i.d. from `generate_rmq_config
223+
:returns: Configuration for Neon MQ-Connector
224+
"""
225+
mq_user_mapping_file = join(dirname(__file__), "templates",
226+
"mq_user_mapping.yml")
227+
with open(mq_user_mapping_file) as f:
228+
mq_user_mapping = yaml.safe_load(f)
229+
230+
mq_config = dict()
231+
LOG.debug(rmq_config.keys())
232+
for user in rmq_config['users']:
233+
username = user['name']
234+
for service in mq_user_mapping.get(username, []):
235+
mq_config[service] = {"user": username,
236+
"password": user['password']}
237+
return mq_config
238+
239+
240+
def configure_backend(username: str = None,
241+
password: str = None,
242+
output_path: str = None):
243+
"""
244+
Generate DIANA backend definitions
245+
@param username: RabbitMQ Admin username to configure
246+
@param password: RabbitMQ Admin password to configure
247+
@param output_path: directory to write output definitions to
248+
"""
249+
from neon_diana_utils.kubernetes_utils import create_github_secret
250+
251+
# Validate output paths
252+
output_path = expanduser(output_path or join(xdg_config_home(), "diana"))
253+
if not validate_output_path(output_path):
254+
click.echo(f"Path exists: {output_path}")
255+
return
256+
257+
# Get Helm charts in output directory for deployment
258+
shutil.copytree(join(dirname(__file__), "helm_charts"),
259+
join(output_path))
260+
chart_path = join(output_path, "diana-backend")
261+
# Cleanup any leftover build files
262+
for root, _, files in walk(output_path):
263+
for file in files:
264+
if any((file.endswith(x) for x in (".lock", ".tgz"))):
265+
remove(join(root, file))
266+
try:
267+
# Generate RabbitMQ config
268+
username = username or click.prompt("RabbitMQ Admin Username", type=str)
269+
password = password or click.prompt("RabbitMQ Admin Password", type=str,
270+
hide_input=True)
271+
rmq_file = join(chart_path, "rabbitmq.json")
272+
rmq_config = generate_rmq_config(username, password, rmq_file)
273+
click.echo(f"Generated RabbitMQ config at {chart_path}/rabbitmq.json")
274+
275+
# Generate MQ Auth config
276+
mq_auth_config = generate_mq_auth_config(rmq_config)
277+
click.echo(f"Generated auth for services: {set(mq_auth_config.keys())}")
278+
279+
# Generate GH Auth config secret
280+
if click.confirm("Configure GitHub token for private services?"):
281+
gh_username = click.prompt("GitHub username", type=str)
282+
gh_token = click.prompt("GitHub Token with `read:packages` "
283+
"permission", type=str)
284+
gh_secret_path = join(chart_path, "templates",
285+
"secret_gh_token.yaml")
286+
create_github_secret(gh_username, gh_token, gh_secret_path)
287+
click.echo(f"Generated GH secret at {gh_secret_path}")
288+
289+
# Generate `diana.yaml` output
290+
keys_config = make_keys_config(False)
291+
config = {**{"MQ": {"users": mq_auth_config,
292+
"server": "neon-rabbitmq",
293+
"port": 5672}},
294+
**keys_config}
295+
output_file = join(chart_path, "diana.yaml")
296+
click.echo(f"Writing configuration to {output_file}")
297+
with open(output_file, 'w+') as f:
298+
yaml.dump(config, f)
299+
click.echo(f"Helm charts generated in {output_path}")
300+
except Exception as e:
301+
click.echo(e)

‎neon_diana_utils/helm_charts/diana-backend/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,5 @@ dependencies:
3535
version: 0.0.1
3636
repository: file://../http-services
3737
- name: diana-mq
38-
version: 0.0.2
38+
version: 0.0.3
3939
repository: file://../mq-services

‎neon_diana_utils/helm_charts/http-services/charts/base-http/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,5 @@ name: base-http
33
description: Library chart for basic HTTP Services
44
type: library
55

6-
version: 0.0.1
6+
version: 0.0.2
77
appVersion: "0.0.6a11"

‎neon_diana_utils/helm_charts/http-services/charts/base-http/templates/_deployment.tpl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ spec:
1313
type: Recreate
1414
template:
1515
metadata:
16+
annotations:
17+
releaseTime: {{ dateInZone "2006-01-02 15:04:05Z" (now) "UTC"| quote }}
1618
labels:
1719
neon.diana.service: {{ .Chart.Name }}
1820
neon.project.name: diana

‎neon_diana_utils/helm_charts/http-services/charts/libretranslate/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,5 @@ appVersion: "0.0.6a11"
88

99
dependencies:
1010
- name: base-http
11-
version: 0.0.1
11+
version: 0.0.2
1212
repository: file://../base-http

‎neon_diana_utils/helm_charts/http-services/charts/tts-coqui/Chart.yaml‎

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,11 @@ apiVersion: v2
22
name: tts-coqui
33
description: Deploy a Coqui TTS Server
44

5-
# A chart can be either an 'application' or a 'library' chart.
6-
#
7-
# Application charts are a collection of templates that can be packaged into versioned archives
8-
# to be deployed.
9-
#
10-
# Library charts provide useful utilities or functions for the chart developer. They're included as
11-
# a dependency of application charts to inject those utilities and functions into the rendering
12-
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
135
type: application
14-
15-
# This is the chart version. This version number should be incremented each time you make changes
16-
# to the chart and its templates, including the app version.
17-
# Versions are expected to follow Semantic Versioning (https://semver.org/)
186
version: 0.0.1
19-
20-
# This is the version number of the application being deployed. This version number should be
21-
# incremented each time you make changes to the application. Versions are not expected to
22-
# follow Semantic Versioning. They should reflect the version the application is using.
23-
# It is recommended to use it with quotes.
247
appVersion: "0.0.6a11"
8+
9+
dependencies:
10+
- name: base-http
11+
version: 0.0.2
12+
repository: file://../base-http

0 commit comments

Comments
 (0)