Skip to content

Commit 7433c05

Browse files
authored
Add GitHub Action testing (#2)
* Add CI tests * Add handler tests * Use String sources for Sitemap Use string as XML ElementTree source to pass CI tests * Update sitemap_dir env variable * Update test_handler.py * Update test.yml
1 parent 5d79278 commit 7433c05

9 files changed

Lines changed: 258 additions & 34 deletions

File tree

.github/workflows/test.yml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Run tests
2+
3+
on: [push]
4+
5+
jobs:
6+
main:
7+
runs-on: ubuntu-20.04
8+
strategy:
9+
matrix:
10+
include:
11+
- python-version: 3.8
12+
- python-version: 3.9
13+
14+
steps:
15+
- uses: actions/checkout@v3
16+
- name: Checkout Geoconnex Namespace
17+
run: |
18+
git clone -b master https://github.com/internetofwater/geoconnex.us.git geoconnex.us
19+
- uses: actions/setup-python@v2
20+
name: Setup Python ${{ matrix.python-version }}
21+
with:
22+
python-version: ${{ matrix.python-version }}
23+
- name: Install requirements 📦
24+
env:
25+
SOURCE_REPO: geoconnex.us
26+
run: |
27+
pip3 install -r requirements.txt
28+
pip3 install -r requirements-dev.txt
29+
python3 setup.py install
30+
- name: run unit tests ⚙️
31+
env:
32+
SOURCE_REPO: geoconnex.us
33+
SITEMAP_DIR: sitemap
34+
run: |
35+
pytest tests/test_util.py
36+
pytest tests/test_handler.py
37+
- name: run flake8 ⚙️
38+
run: |
39+
find . -type f -name "*.py" -not -path "./geoconnex.us/*" | xargs flake8

requirements-dev.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
flake8
2+
pytest

setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ def get_package_version():
118118
project_urls={
119119
'Homepage': 'https://github.com/cgs-earth/sitemap-generator',
120120
'Source Code': 'https://github.com/cgs-earth/sitemap-generator',
121-
'Issue Tracker': 'https://github.com/cgs-earth/sitemap-generator/issues'
121+
'Issue Tracker': 'https://github.com/cgs-earth/sitemap-generator/issues' # noqa
122122
},
123123
cmdclass={'test': PyTest},
124124
test_suite='tests.run_tests'

sitemap_generator/handler.py

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,10 @@
3636
from pathlib import Path
3737
from shutil import copy2
3838
from typing import Iterator
39-
from xml.etree.ElementTree import fromstring
4039

41-
from sitemap_generator.util import (url_join, get_sitemapindex,
42-
SITEMAPINDEX_FOREACH, get_urlset,
43-
URLSET_FOREACH, walk_path,
40+
from sitemap_generator.util import (url_join, get_smi, add_smi_node,
41+
get_urlset, add_urlset_node,
42+
write_tree, walk_path,
4443
parse, OPTION_VERBOSITY)
4544

4645
LOGGER = logging.getLogger(__name__)
@@ -53,11 +52,8 @@
5352
TREE = REPO.heads.master.commit.tree
5453
NAMESPACE = TREE / SOURCE_REPO_PATH
5554

56-
# Environment var for sitemap output
57-
SITEMAP_LOC = os.environ.get('SITEMAP_LOC', '/sitemap')
5855
# Sitemap directory objects
59-
SITEMAP_DIR = Path(SITEMAP_LOC)
60-
SITEMAP_ARGS = {'encoding': 'utf-8', 'xml_declaration': True}
56+
SITEMAP_DIR = Path(os.environ.get('SITEMAP_DIR', '/sitemap'))
6157

6258

6359
class Handler:
@@ -75,7 +71,7 @@ def __init__(self, filepath: Path, uri_stem: str) -> None:
7571
self.root_path = filepath
7672
self.uri_stem = uri_stem
7773

78-
def handle(self, ) -> None:
74+
def handle(self) -> None:
7975
"""
8076
Handle sitemap creation sitemapindex
8177
@@ -112,13 +108,12 @@ def make_urlset(self, filename: Path) -> None:
112108
if '$' in url_:
113109
LOGGER.warning(f'Regex detected in {filename}')
114110
return
115-
urlitem = URLSET_FOREACH.format(url_, file_time)
116-
root.append(fromstring(urlitem))
111+
add_urlset_node(root, url_, file_time)
117112

118113
# Write sitemap.xml
119114
fidx = f'{filename.stem}__{i}'
120115
sitemap_file = (filename.parent / fidx).with_suffix('.xml')
121-
tree.write(sitemap_file, **SITEMAP_ARGS)
116+
write_tree(tree, sitemap_file)
122117

123118
def make_sitemap(self, files: Iterator[Path]) -> None:
124119
"""
@@ -128,7 +123,7 @@ def make_sitemap(self, files: Iterator[Path]) -> None:
128123
129124
:returns: `None`
130125
"""
131-
tree, root = get_sitemapindex()
126+
tree, root = get_smi()
132127
for f in files:
133128
LOGGER.debug(f'Processing urlset: {f.resolve()}')
134129

@@ -148,13 +143,12 @@ def make_sitemap(self, files: Iterator[Path]) -> None:
148143

149144
# create to link /sitemap/_sitemap.xml
150145
file_time = self._get_filetime(file_path)
151-
url_ = url_join(self.uri_stem, str(file_path))
152-
sitemapitem = SITEMAPINDEX_FOREACH.format(url_, file_time)
153-
root.append(fromstring(sitemapitem))
146+
url_ = url_join(self.uri_stem, file_path)
147+
add_smi_node(root, url_, file_time)
154148

155149
sitemap_out = SITEMAP_DIR / '_sitemap.xml'
156150
LOGGER.debug(f'Writing sitemapindex to {sitemap_out}')
157-
tree.write(sitemap_out, **SITEMAP_ARGS)
151+
write_tree(tree, sitemap_out)
158152

159153
def _get_filetime(self, filename: Path) -> str:
160154
"""

sitemap_generator/models/sitemapindex.xml

Lines changed: 0 additions & 3 deletions
This file was deleted.

sitemap_generator/models/urlset.xml

Lines changed: 0 additions & 3 deletions
This file was deleted.

sitemap_generator/util.py

Lines changed: 39 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,21 @@
3838

3939
LOGGER = logging.getLogger(__name__)
4040

41-
MODELS = Path(__file__).parent.resolve() / 'models'
42-
41+
SITEMAP_ARGS = {'encoding': 'utf-8', 'xml_declaration': True}
42+
SITEMAPINDEX = '''<?xml version="1.0"?>
43+
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
44+
</sitemapindex>
45+
'''
4346
SITEMAPINDEX_FOREACH = '''
4447
<sitemap>
4548
<loc>{}</loc>
4649
<lastmod>{}</lastmod>
4750
</sitemap>
4851
'''
52+
URLSET = '''<?xml version="1.0"?>
53+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
54+
</urlset>
55+
'''
4956
URLSET_FOREACH = '''
5057
<url>
5158
<loc>{}</loc>
@@ -54,16 +61,38 @@
5461
'''
5562

5663

57-
def get_sitemapindex():
58-
tree = ET.parse(MODELS / 'sitemapindex.xml')
59-
ET.indent(tree, ' ')
60-
return tree, tree.getroot()
64+
def write_tree(tree, file):
65+
tree.write(file, **SITEMAP_ARGS)
66+
67+
68+
def get_smi():
69+
root = ET.fromstring(SITEMAPINDEX)
70+
tree = ET.ElementTree(root)
71+
try:
72+
ET.indent(tree)
73+
except AttributeError:
74+
LOGGER.warning('Unable to indent')
75+
return tree, root
76+
77+
78+
def add_smi_node(node, loc, lastmod):
79+
_ = SITEMAPINDEX_FOREACH.format(loc, lastmod)
80+
node.append(ET.fromstring(_))
6181

6282

6383
def get_urlset():
64-
tree = ET.parse(MODELS / 'urlset.xml')
65-
ET.indent(tree, ' ')
66-
return tree, tree.getroot()
84+
root = ET.fromstring(URLSET)
85+
tree = ET.ElementTree(root)
86+
try:
87+
ET.indent(tree)
88+
except AttributeError:
89+
LOGGER.warning('Unable to indent')
90+
return tree, root
91+
92+
93+
def add_urlset_node(node, loc, lastmod):
94+
_ = URLSET_FOREACH.format(loc, lastmod)
95+
node.append(ET.fromstring(_))
6796

6897

6998
def walk_path(path: Path, regex: str) -> Iterator[Path]:
@@ -96,7 +125,7 @@ def url_join(*parts):
96125
97126
:returns: str of resulting URL
98127
"""
99-
return '/'.join([p.strip().strip('/') for p in parts])
128+
return '/'.join([str(p).strip().strip('/') for p in parts])
100129

101130

102131
def parse(filename: Path, n: int = 50000) -> list:

tests/test_handler.py

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
# =================================================================
2+
#
3+
# Authors: Benjamin Webb <bwebb@lincolninst.edu>
4+
#
5+
# Copyright (c) 2023 Benjamin Webb
6+
#
7+
# Permission is hereby granted, free of charge, to any person
8+
# obtaining a copy of this software and associated documentation
9+
# files (the "Software"), to deal in the Software without
10+
# restriction, including without limitation the rights to use,
11+
# copy, modify, merge, publish, distribute, sublicense, and/or sell
12+
# copies of the Software, and to permit persons to whom the
13+
# Software is furnished to do so, subject to the following
14+
# conditions:
15+
#
16+
# The above copyright notice and this permission notice shall be
17+
# included in all copies or substantial portions of the Software.
18+
#
19+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
21+
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
23+
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
24+
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25+
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
26+
# OTHER DEALINGS IN THE SOFTWARE.
27+
#
28+
# =================================================================
29+
30+
from datetime import datetime
31+
from pathlib import Path
32+
import xml.etree.ElementTree as ET
33+
34+
from sitemap_generator.handler import Handler, SITEMAP_DIR
35+
from sitemap_generator.util import walk_path, url_join
36+
37+
THIS_DIR = Path(__file__).parent.resolve()
38+
NAMESPACE = THIS_DIR / 'data' / 'namespaces'
39+
40+
URI_STEM = 'https://geoconnex.us'
41+
HANDLER = Handler(str(NAMESPACE), URI_STEM)
42+
43+
44+
def test_handler():
45+
HANDLER.handle()
46+
glob = list(walk_path(SITEMAP_DIR, r'.*'))
47+
glob1 = list(walk_path(SITEMAP_DIR, r'.*xml'))
48+
assert len(glob) == len(glob1)
49+
50+
51+
def test_sitemapindex():
52+
[sitemapindex] = list(walk_path(SITEMAP_DIR, r'.*_sitemap.xml'))
53+
assert sitemapindex.name == '_sitemap.xml'
54+
assert HANDLER._get_rel_path(sitemapindex) == '.'
55+
56+
_ = HANDLER._get_filetime(sitemapindex)
57+
file_time = datetime.strptime(_, '%Y-%m-%dT%H:%M:%SZ')
58+
today = datetime.utcnow().strftime('%Y-%m-%d')
59+
assert file_time.strftime('%Y-%m-%d') == today
60+
61+
tree = ET.parse(sitemapindex)
62+
root = tree.getroot()
63+
64+
assert all(child.tag == 'sitemap' for child in root)
65+
assert all(URI_STEM in child.find('loc').text for child in root)
66+
67+
links = root.find('sitemap')
68+
_ = links.find('lastmod').text
69+
lastmod = datetime.strptime(_, '%Y-%m-%dT%H:%M:%SZ')
70+
assert lastmod.strftime('%Y-%m-%d') != today
71+
72+
73+
def test_urlset():
74+
[urlset] = list(walk_path(SITEMAP_DIR, r'.*links__0.xml'))
75+
assert urlset.name == 'links__0.xml'
76+
assert HANDLER._get_rel_path(urlset) == 'iow'
77+
78+
_ = HANDLER._get_filetime(urlset)
79+
file_time = datetime.strptime(_, '%Y-%m-%dT%H:%M:%SZ')
80+
today = datetime.utcnow().strftime('%Y-%m-%d')
81+
assert file_time.strftime('%Y-%m-%d') != today
82+
83+
namespace = url_join(URI_STEM, HANDLER._get_rel_path(urlset))
84+
assert namespace == 'https://geoconnex.us/iow'
85+
86+
[urlset] = list(walk_path(SITEMAP_DIR, r'.*autotest1__0.xml'))
87+
file_time = HANDLER._get_filetime(urlset)
88+
tree = ET.parse(urlset)
89+
root = tree.getroot()
90+
91+
assert all(child.tag == 'url' for child in root)
92+
for child in root:
93+
assert file_time == child.find('lastmod').text
94+
print(child.find('lastmod').text, child.find('loc').text)
95+
assert all(file_time == child.find('lastmod').text for child in root)
96+
97+
url = root.find('url')
98+
lastmod = url.find('lastmod').text
99+
assert lastmod == '2023-06-23T22:38:13Z'

tests/test_util.py

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# =================================================================
2+
#
3+
# Authors: Benjamin Webb <bwebb@lincolninst.edu>
4+
#
5+
# Copyright (c) 2023 Benjamin Webb
6+
#
7+
# Permission is hereby granted, free of charge, to any person
8+
# obtaining a copy of this software and associated documentation
9+
# files (the "Software"), to deal in the Software without
10+
# restriction, including without limitation the rights to use,
11+
# copy, modify, merge, publish, distribute, sublicense, and/or sell
12+
# copies of the Software, and to permit persons to whom the
13+
# Software is furnished to do so, subject to the following
14+
# conditions:
15+
#
16+
# The above copyright notice and this permission notice shall be
17+
# included in all copies or substantial portions of the Software.
18+
#
19+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
21+
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
23+
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
24+
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25+
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
26+
# OTHER DEALINGS IN THE SOFTWARE.
27+
#
28+
# =================================================================
29+
30+
from pathlib import Path
31+
32+
from sitemap_generator import util
33+
34+
THIS_DIR = Path(__file__).parent.resolve()
35+
NAMESPACE = THIS_DIR / 'data' / 'namespaces'
36+
37+
38+
def test_walk_path():
39+
glob = util.walk_path(NAMESPACE, r'.*')
40+
assert len(list(glob)) == 3
41+
42+
glob = util.walk_path(NAMESPACE, r'.*csv')
43+
assert len(list(glob)) == 2
44+
45+
glob = util.walk_path(NAMESPACE, r'.*xml')
46+
assert len(list(glob)) == 1
47+
48+
49+
def test_parse_and_chunk():
50+
glob = util.walk_path(NAMESPACE / 'ref', r'.*csv')
51+
hu08 = next(glob)
52+
assert hu08.stem == 'hu08'
53+
54+
lines = util.parse(hu08)
55+
assert len(lines) == 1
56+
assert len(lines[-1]) == 2399
57+
58+
chunks = util.parse(hu08, 1000)
59+
assert len(chunks) == 3
60+
assert len(chunks[0]) == 1000
61+
assert len(chunks[-1]) == 399
62+
63+
[lines] = util.parse(hu08)
64+
chunks = util.chunkify(lines, 100)
65+
assert len(chunks) == 24
66+
assert len(chunks[0]) == 100
67+
assert len(chunks[-1]) == 99

0 commit comments

Comments
 (0)