Skip to content

Commit 7bdb961

Browse files
authored
Merge pull request #130 from AIRInstitute/develop
Update branch
2 parents ae24fc5 + 2bf01a1 commit 7bdb961

10 files changed

Lines changed: 204 additions & 38 deletions

File tree

data/postgresIngestDB/init.sql

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,10 +90,6 @@ CREATE TABLE IF NOT EXISTS segment_data (
9090
total_big_birds INT,
9191
frames JSONB,
9292
received_at TIMESTAMP NOT NULL DEFAULT now(),
93-
CONSTRAINT fk_camera_segment
94-
FOREIGN KEY (camera_id)
95-
REFERENCES bird_statistics(camera_id)
96-
ON DELETE CASCADE
9793
);
9894

9995
-- Imágenes de mapas de calor

docker-compose.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ services:
100100
- /etc/envs/.env
101101
volumes:
102102
- /home/bisite/exclusion_eolica.csv:/home/exclusion_eolica.csv
103+
- ./ia4birds-all/ai4birds-ingest-service/ai4birds_ingest_service/heatmaps:/app/ai4birds_ingest_service/heatmaps
103104
ports:
104105
- 5002:5000
105106
networks:

ia4birds-all/ai4birds-ingest-service/ai4birds_ingest_service/api/__init__.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Copyright 2023 AIRInstitute
33
# See LICENSE for details.
44
# Author: AIRInstitute (@AIRInstitute on GitHub)
5-
from ai4birds_ingest_service.api.namespaces.ingest_ns import ns_ebird, ns_xenocanto, ns_windmap, ns_exclusionmap, ns_sensitivity, ns_dataBird, ns_device_status, ns_segment_data, ns_heatmap_data, ns_bird_statistics
5+
from ai4birds_ingest_service.api.namespaces.ingest_ns import ns_ebird, ns_xenocanto, ns_windmap, ns_exclusionmap,ns_sensitivity, ns_dataBird, ns_device_status, ns_segment_data, ns_heatmap_data, ns_bird_statistics, ns_heatmap_files
66

77
__author__ = 'AIRInstitute'
88
__version__ = '1.0'
@@ -18,4 +18,5 @@
1818
namespaces.append(ns_device_status)
1919
namespaces.append(ns_segment_data)
2020
namespaces.append(ns_heatmap_data)
21-
namespaces.append(ns_bird_statistics)
21+
namespaces.append(ns_bird_statistics)
22+
namespaces.append(ns_heatmap_files)

ia4birds-all/ai4birds-ingest-service/ai4birds_ingest_service/api/namespaces/ingest_ns.py

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import flask
2+
import os
23
from flask import jsonify, request as flask_request, send_file, Response
34
from flask_restx import Resource
45

@@ -38,7 +39,7 @@
3839
from ai4birds_ingest_service.services.data_segment_service import DataSegmentService
3940
from ai4birds_ingest_service.services.data_heatmap_service import DataHeatmapService
4041
from ai4birds_ingest_service.services.bird_statistics_service import BirdStatisticsService
41-
42+
from ai4birds_ingest_service.services.heatmap_files_service import HeatmapFilesService
4243
# Define namespaces
4344
ns_xenocanto = api.namespace('xenocanto', description='Xenocanto requests')
4445
ns_ebird = api.namespace('ebird', description='eBird requests')
@@ -50,7 +51,7 @@
5051
ns_segment_data = api.namespace('segment-data', description='Segment data operations')
5152
ns_heatmap_data = api.namespace('heatmap-data', description='Heatmap data operations')
5253
ns_bird_statistics = api.namespace('bird-statistics', description='Bird statistics operations')
53-
54+
ns_heatmap_files = api.namespace('heatmap-files', description='Heatmap files operations')
5455

5556
@ns_dataBird.route('/')
5657
class DataBird(Resource):
@@ -328,4 +329,55 @@ def get(self):
328329
return {'camera_statistics': data}, status_code
329330
except:
330331
return handle500error(ns_bird_statistics)
332+
333+
334+
@ns_heatmap_files.route('/download/<string:filename>')
335+
class HeatmapFileDownload(Resource):
336+
"""
337+
Returns heatmap file as direct download/streaming response.
338+
"""
339+
@limiter.limit('1000000/hour')
340+
def get(self, filename):
341+
"""
342+
Download a specific heatmap file directly.
343+
344+
Args:
345+
filename (str): Name of the heatmap file to download
346+
347+
Returns:
348+
Flask Response: Direct file response or error
349+
"""
350+
try:
351+
service = HeatmapFilesService()
352+
return service.get_heatmap_file_response(filename)
353+
354+
except Exception as e:
355+
logger.error(f"HeatmapFileDownload Error: {e}")
356+
return {'error': str(e)}, 500
357+
358+
359+
@ns_heatmap_files.route('/files')
360+
class HeatmapFilesList(Resource):
361+
"""
362+
Lists all available heatmap files in the directory.
363+
"""
364+
@limiter.limit('1000000/hour')
365+
def get(self):
366+
"""
367+
Get a list of all available heatmap files.
368+
369+
Returns:
370+
dict: List of available heatmap files with metadata
371+
"""
372+
try:
373+
logger.info("Creating HeatmapFilesService instance in endpoint")
374+
service = HeatmapFilesService()
375+
logger.info(f"Service directory: {service.heatmaps_directory}")
376+
logger.info(f"Directory exists: {os.path.exists(service.heatmaps_directory)}")
377+
result, status_code = service.list_heatmap_files()
378+
return result, status_code
379+
380+
except Exception as e:
381+
logger.error(f"HeatmapFilesList Error: {e}")
382+
return {'error': str(e)}, 500
331383

ia4birds-all/ai4birds-ingest-service/ai4birds_ingest_service/config.py

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,9 @@
1111
load_dotenv(dotenv_path)
1212

1313
# api config
14-
# PORT = 5001
15-
PORT = 5000
16-
HOST = '0.0.0.0'
17-
URL_PREFIX = '/ai4birds-ingest-service/v1'
14+
PORT = os.getenv('PORT_INGEST', 5000)
15+
HOST = os.getenv('HOST_INGEST', '0.0.0.0')
16+
URL_PREFIX = os.getenv('URL_PREFIX_INGEST', '/ai4birds-ingest-service/v1')
1817
DEBUG_MODE = True
1918

2019
#DB config
@@ -26,7 +25,6 @@
2625
'database' : os.getenv('POSTGRES_INGEST_DB')
2726
}
2827

29-
3028
BACKEND_URL = os.getenv('BACKEND_URL')
3129

3230
SPECIES_LIST = {
@@ -58,10 +56,17 @@
5856

5957
EXCLUSION_EOLICA_CSV_PATH = '/home/exclusion_eolica.csv'
6058
API_SPEC_PATH = '/app/ai4birds_ingest_service/doc/api-spec.yaml'
59+
HEATMAP_PATH = "./ai4birds_ingest_service/heatmaps"
60+
HEATMAP_ENDPOINT = os.getenv('HEATMAP_ENDPOINT', 'http://localhost:5002/ai4birds-ingest-service/v1/heatmap-files/download/')
61+
62+
# MQTT BROKER TEST
63+
# MQTT_BROKER = 'broker.hivemq.com'
64+
# MQTT_PORT = 1883
6165

6266
# MQTT BROKER
6367
MQTT_BROKER = os.getenv('MQTT_HOST', 'localhost')
6468
MQTT_PORT = int(os.getenv('MQTT_PORT', 1883))
69+
6570
MQTT_KEEPALIVE = 60
6671
MQTT_TLS_ENABLED = False
6772
TIME_WITHOUT_MESSAGE = 60

ia4birds-all/ai4birds-ingest-service/ai4birds_ingest_service/model/data_heatmap/data_heatmap.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from datetime import datetime
66
from typing import Dict, Any
77

8-
from ai4birds_ingest_service import logger
8+
from ai4birds_ingest_service import config, logger
99

1010
class DataHeatmap:
1111
def __init__(self, camera_id: int, heatmap_for: str, image_url: str, generated_at: str) -> None:
@@ -31,16 +31,16 @@ def from_dict(cls, data: Dict[str, Any], image_bytes: bytes) -> 'DataHeatmap':
3131
heatmap_for = data.get('heatmap_for', 'Unknown')
3232

3333
#determine the root path of the project
34-
project_root = './ai4birds_ingest_service/'
35-
heatmap_dir = os.path.join(project_root, "heatmaps")
36-
os.makedirs(heatmap_dir, exist_ok=True)
34+
os.makedirs(config.HEATMAP_PATH, exist_ok=True)
3735

3836
timestamp = datatime.strftime('%Y%m%d_%H%M%S')
39-
filename = f"{camera_id}_{heatmap_for}_{timestamp}.png"
37+
filename = f"{camera_id}_{timestamp}.png"
4038

4139
# Path
42-
image_abs_path = os.path.join(heatmap_dir, filename)
43-
image_rel_path = os.path.relpath(image_abs_path, start=project_root)
40+
image_abs_path = os.path.join(config.HEATMAP_PATH, filename)
41+
image_rel_path = config.HEATMAP_ENDPOINT + filename
42+
43+
logger.info(f"image_rel_path: {image_rel_path}")
4444

4545
# Save image
4646
with open(image_abs_path, 'wb') as f:

ia4birds-all/ai4birds-ingest-service/ai4birds_ingest_service/model/data_segment/data_segment.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,13 @@ def from_dict(cls, data: Dict[str, Any]) -> 'DataSegment':
3636

3737
try:
3838
return cls(
39-
camera_id = data.get('calibration', {}).get('camera_id', None),
40-
segment_idx = data.get('calibration', {}).get('segment_idx', None),
41-
colatitude = data.get('calibration', {}).get('absolute_colatitude', None),
42-
azimuth = data.get('calibration', {}).get('absolute_azimuth', None),
43-
zoom_level = data.get('calibration', {}).get('zoom_level', None),
44-
average_area = data.get('calibration', {}).get('average_area', None),
45-
total_big_birds = data.get('calibration', {}).get('total_big_birds', None),
39+
camera_id = data.get('camera_id', None),
40+
segment_idx = data.get('segment_idx', None),
41+
colatitude = data.get('absolute_colatitude', None),
42+
azimuth = data.get('absolute_azimuth', None),
43+
zoom_level = data.get('zoom_level', None),
44+
average_area = data.get('average_area', None),
45+
total_big_birds = data.get('total_big_birds', None),
4646
frames = json.dumps(data.get('frames', {})),
4747
received_at = datatime
4848
)
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
import os
2+
import mimetypes
3+
from flask import Response, send_file
4+
from ai4birds_ingest_service import config, logger
5+
6+
class HeatmapFilesService:
7+
"""Service for handling heatmap files operations."""
8+
9+
def __init__(self):
10+
# Directorio donde se almacenan las imágenes de mapas de calor
11+
self.heatmaps_directory = '/app/ai4birds_ingest_service/heatmaps'
12+
logger.info(f"Initialized HeatmapFilesService with directory: {self.heatmaps_directory}")
13+
logger.info(f"Directory exists: {os.path.exists(self.heatmaps_directory)}")
14+
logger.info(f"Is directory: {os.path.isdir(self.heatmaps_directory)}")
15+
if os.path.exists(self.heatmaps_directory):
16+
logger.info(f"Contents: {os.listdir(self.heatmaps_directory)}")
17+
18+
def get_heatmap_file_response(self, filename: str):
19+
"""Get heatmap file as HTTP response for direct file serving.
20+
21+
Args:
22+
filename (str): Name of the heatmap file
23+
24+
Returns:
25+
Flask Response: File response or error response
26+
"""
27+
logger.info(f"Requesting heatmap file response: {filename}")
28+
29+
try:
30+
# Validar que el archivo existe
31+
if not self._file_exists(filename):
32+
logger.warning(f"Heatmap file not found: {filename}")
33+
return Response("File not found", status=404)
34+
35+
# Validar que es una imagen válida
36+
if not self._is_valid_image_file(filename):
37+
logger.warning(f"Invalid file type requested: {filename}")
38+
return Response("Invalid file type. Only image files are allowed", status=400)
39+
40+
# Servir el archivo directamente
41+
file_path = os.path.join(self.heatmaps_directory, filename)
42+
43+
# Obtener el tipo MIME
44+
mime_type, _ = mimetypes.guess_type(filename)
45+
if not mime_type:
46+
mime_type = 'application/octet-stream'
47+
48+
logger.info(f"Serving heatmap file response: {filename}")
49+
return send_file(
50+
file_path,
51+
mimetype=mime_type,
52+
as_attachment=False,
53+
download_name=filename
54+
)
55+
56+
except Exception as e:
57+
logger.error(f"Error serving heatmap file response {filename}: {e}")
58+
return Response(f"Internal server error: {str(e)}", status=500)
59+
60+
def _file_exists(self, filename: str) -> bool:
61+
"""Check if file exists in heatmaps directory.
62+
63+
Args:
64+
filename (str): Name of the file to check
65+
66+
Returns:
67+
bool: True if file exists, False otherwise
68+
"""
69+
file_path = os.path.join(self.heatmaps_directory, filename)
70+
return os.path.isfile(file_path)
71+
72+
def _is_valid_image_file(self, filename: str) -> bool:
73+
"""Check if file has a valid image extension.
74+
75+
Args:
76+
filename (str): Name of the file to check
77+
78+
Returns:
79+
bool: True if file has valid image extension, False otherwise
80+
"""
81+
valid_extensions = {'.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp', '.svg'}
82+
file_extension = os.path.splitext(filename.lower())[1]
83+
return file_extension in valid_extensions
84+
85+
def list_heatmap_files(self):
86+
"""List all available heatmap files.
87+
88+
Returns:
89+
tuple: List of files and status code
90+
"""
91+
logger.info("Listing all heatmap files")
92+
93+
try:
94+
if not os.path.exists(self.heatmaps_directory):
95+
logger.warning("Heatmaps directory does not exist")
96+
return {"files": [], "message": "Heatmaps directory not found"}, 404
97+
98+
files = []
99+
for filename in os.listdir(self.heatmaps_directory):
100+
file_path = os.path.join(self.heatmaps_directory, filename)
101+
if os.path.isfile(file_path) and self._is_valid_image_file(filename):
102+
files.append({
103+
"filename": filename,
104+
"size": os.path.getsize(file_path),
105+
"modified": os.path.getmtime(file_path)
106+
})
107+
108+
logger.info(f"Found {len(files)} heatmap files")
109+
return {"files": files, "count": len(files)}, 200
110+
111+
except Exception as e:
112+
logger.error(f"Error listing heatmap files: {e}")
113+
return {"error": f"Failed to list files: {str(e)}"}, 500

ia4birds-all/ai4birds-ingest-service/ai4birds_ingest_service/tests/test_data/test_data.json

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,14 @@
11
{
22
"segment_data": [
33
{
4-
"calibration": {
5-
"camera_id": "CAM123",
6-
"segment_idx": 1,
7-
"absolute_colatitude": 92.25,
8-
"absolute_azimuth": 87.50999999999999,
9-
"zoom_level": 17,
10-
"average_area": 1941.2667833562925,
11-
"total_big_birds": 310,
12-
"horario": "Morning"
13-
},
4+
"camera_id": "CAM123",
5+
"segment_idx": 1,
6+
"absolute_colatitude": 92.25,
7+
"absolute_azimuth": 87.50999999999999,
8+
"zoom_level": 17,
9+
"average_area": 1941.2667833562925,
10+
"total_big_birds": 310,
11+
"horario": "Morning",
1412
"frames": {
1513
"1357": [],
1614
"1358": [

ia4birds-all/ai4birds-ingest-service/ai4birds_ingest_service/tests/test_integration.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ def test_bird_statistics_update(mqtt_client, test_payload):
6262
db = PostgresSingleton.getInstance()
6363
db.connect()
6464

65-
camera_id = test_payload["calibration"]["camera_id"]
65+
camera_id = test_payload["camera_id"]
6666
frames = test_payload["frames"]
6767

6868
bird_names_detected = set()

0 commit comments

Comments
 (0)