|
| 1 | +#!/usr/bin/env python3 |
| 2 | +import os |
| 3 | +import pytest |
| 4 | +import requests |
| 5 | +import shutil |
| 6 | +import time |
| 7 | + |
| 8 | +BASE_URL = os.getenv('TEST_BASE_URL', 'http://localhost:6800') |
| 9 | +RUN_PROJECT = os.getenv('TEST_RUN_PROJECT', 'example') |
| 10 | +RUN_VERSION = os.getenv('TEST_RUN_VERSION', 'latest') |
| 11 | +RUN_SPIDER = os.getenv('TEST_RUN_SPIDER', 'static') |
| 12 | +MAX_WAIT = int(os.getenv('TEST_MAX_WAIT', '6')) |
| 13 | +STATIC_SLEEP = float(os.getenv('TEST_STATIC_SLEEP', '2')) |
| 14 | +WITH_K8S = bool(os.getenv('TEST_WITH_K8S')) |
| 15 | +LOG_BASE = os.getenv('TEST_LOG_BASE', '/tmp/joblogs') |
| 16 | + |
| 17 | +LOG_DIR_LIVE = os.path.join(LOG_BASE, 'live') |
| 18 | +LOG_DIR_CONT = os.path.join(LOG_BASE, 'container', 'a') |
| 19 | + |
| 20 | +@pytest.mark.skipif(not WITH_K8S, reason="joblogs only implemented for Kubernetes") |
| 21 | +def test_live(): |
| 22 | + # Schedule a job |
| 23 | + response = requests.post(BASE_URL + '/schedule.json', data={ |
| 24 | + 'project': RUN_PROJECT, 'spider': RUN_SPIDER, '_version': RUN_VERSION, |
| 25 | + 'setting': 'STATIC_SLEEP=%d' % STATIC_SLEEP |
| 26 | + }) |
| 27 | + assert_response_ok(response) |
| 28 | + jobid = response.json()['jobid'] |
| 29 | + assert jobid is not None |
| 30 | + # Make sure the job is running |
| 31 | + listjobs_wait(jobid, 'running') |
| 32 | + |
| 33 | + # Check that log file is present and has sensible content |
| 34 | + log_path_live = os.path.join(LOG_DIR_LIVE, jobid + '.txt') |
| 35 | + assert(os.path.isfile(log_path_live)) |
| 36 | + with open(log_path_live, 'r') as f: |
| 37 | + assert('(bot: %s)' % RUN_PROJECT in f.read(2048)) |
| 38 | + |
| 39 | + # Wait until finished |
| 40 | + listjobs_wait(jobid, 'finished', max_wait=STATIC_SLEEP+MAX_WAIT) |
| 41 | + |
| 42 | +@pytest.mark.skipif(not WITH_K8S, reason="joblogs only implemented for Kubernetes") |
| 43 | +def test_container(): |
| 44 | + # Schedule a job |
| 45 | + response = requests.post(BASE_URL + '/schedule.json', data={ |
| 46 | + 'project': RUN_PROJECT, 'spider': RUN_SPIDER, '_version': RUN_VERSION, |
| 47 | + 'setting': 'STATIC_SLEEP=%d' % STATIC_SLEEP |
| 48 | + }) |
| 49 | + assert_response_ok(response) |
| 50 | + jobid = response.json()['jobid'] |
| 51 | + assert jobid is not None |
| 52 | + |
| 53 | + # Wait until finished, and a bit more to finish log processing |
| 54 | + listjobs_wait(jobid, 'finished', max_wait=STATIC_SLEEP+MAX_WAIT) |
| 55 | + time.sleep(1) |
| 56 | + |
| 57 | + # Make sure we find logfile in container storage (configured as local storage) |
| 58 | + log_path_cont = os.path.join(LOG_DIR_CONT, 'logs', RUN_PROJECT, RUN_SPIDER, jobid + '.log') |
| 59 | + assert(os.path.isfile(log_path_cont)) |
| 60 | + with open(log_path_cont, 'r') as f: |
| 61 | + log = f.read() |
| 62 | + # bot name in first lines |
| 63 | + assert('(bot: %s)' % RUN_PROJECT in log) |
| 64 | + # 'Spider finished' in last lines |
| 65 | + assert('Spider closed (finished)' in log) |
| 66 | + |
| 67 | +def assert_response_ok(response): |
| 68 | + assert response.status_code == 200 |
| 69 | + assert response.headers['Content-Type'] == 'application/json' |
| 70 | + assert response.json()['status'] == 'ok' |
| 71 | + |
| 72 | +def listjobs_wait(jobid, state, max_wait=MAX_WAIT): |
| 73 | + started = time.monotonic() |
| 74 | + while time.monotonic() - started < max_wait: |
| 75 | + response = requests.get(BASE_URL + '/listjobs.json') |
| 76 | + assert_response_ok(response) |
| 77 | + for j in response.json()[state]: |
| 78 | + if j['id'] == jobid: |
| 79 | + return True |
| 80 | + time.sleep(0.5) |
| 81 | + assert False, 'Timeout waiting for job state change' |
0 commit comments