New image source europeana

.gitignore

@@ -68,3 +68,4 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+*venv/

serverside/variety_server_options.json

@@ -23,6 +23,10 @@
     "min_fill_queue_interval": 3600,
     "min_download_interval": 1800
   },
+  "europeana": {
+    "min_fill_queue_interval": 1200,
+    "min_download_interval": 1200
+  },
   "status_message": {
     "0.0.1": "",
     "*": ""

tests/TestEuropeanaConfigurableDownloader.py

@@ -0,0 +1,50 @@
+#!/usr/bin/python3
+# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
+### BEGIN LICENSE
+# Copyright (c) 2012, Peter Levi <peterlevi@peterlevi.com>
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+### END LICENSE
+
+import unittest
+
+from tests.TestDownloader import test_download_one_for
+from variety.AttrDict import AttrDict
+from variety.plugins.builtin.downloaders.EuropeanaConfigurableSource import (
+    EuropeanaConfigurableSource,
+)
+
+
+class TestEuropeanaConfigurableDownloader(unittest.TestCase):
+    def _source(self):
+        parent = AttrDict()
+        parent.size_ok = lambda x, y: True
+        source = EuropeanaConfigurableSource()
+        source.set_variety(parent)
+        return source
+
+    def test_download_one(self):
+        test_download_one_for(self, self._source().create_downloader("sea"))
+
+    def test_validate(self):
+        source = self._source()
+        self.assertIsNone(source.validate("")[1])
+        self.assertIsNone(source.validate("forest")[1])
+
+    def test_fill_queue(self):
+        dl = self._source().create_downloader("Rome")
+        queue = dl.fill_queue()
+        self.assertTrue(len(queue) > 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/TestEuropeanaDownloader.py

@@ -0,0 +1,36 @@
+#!/usr/bin/python3
+# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
+### BEGIN LICENSE
+# Copyright (c) 2012, Peter Levi <peterlevi@peterlevi.com>
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+### END LICENSE
+
+import unittest
+
+from tests.TestDownloader import get_plugin_downloader, test_download_one_for
+
+
+class TestEuropeanaDownloader(unittest.TestCase):
+    def test_download_one(self):
+        dl = get_plugin_downloader("EuropeanaDownloader")
+        test_download_one_for(self, dl)
+
+    def test_fill_queue(self):
+        dl = get_plugin_downloader("EuropeanaDownloader")
+        dl.target_folder = "/tmp/variety/"
+        queue = dl.fill_queue()
+        self.assertTrue(len(queue) > 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

variety/Texts.py

@@ -45,6 +45,10 @@
         "High-resolution photos from Unsplash.com",
         _("High-resolution photos from Unsplash.com"),
     ),
+    "europeana": (
+        "Europe's digital cultural heritage from Europeana.eu",
+        _("Europe's digital cultural heritage from Europeana.eu")
+    ),
 }
 
 profile_path = get_profile_path(expanded=False)

variety/plugins/builtin/downloaders/EuropeanaConfigurableSource.py

@@ -0,0 +1,102 @@
+# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
+### BEGIN LICENSE
+# Copyright (c) 2012, Peter Levi <peterlevi@peterlevi.com>
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+### END LICENSE
+import logging
+import random
+
+from requests import HTTPError
+
+from variety.plugins.builtin.downloaders.EuropeanaDownloader import EuropeanaDownloader
+from variety.plugins.downloaders.ConfigurableImageSource import ConfigurableImageSource
+from variety.plugins.downloaders.DefaultDownloader import DefaultDownloader
+from variety.plugins.downloaders.ImageSource import Throttling
+from variety.Util import Util, _
+
+logger = logging.getLogger("variety")
+
+random.seed()
+
+
+class UnsupportedConfig(Exception):
+    pass
+
+
+class EuropeanaConfigurableSource(ConfigurableImageSource):
+    class EuropeanaConfigurableDownloader(EuropeanaDownloader):
+        def __init__(self, source, config):
+            DefaultDownloader.__init__(self, source, config)
+            self.set_variety(source.get_variety())
+
+        def get_source_type(self):
+            return self.source.get_source_type()
+
+        def get_description(self):
+            return self.config
+
+        def get_folder_name(self):
+            return super(DefaultDownloader, self).get_folder_name()
+
+        def get_europeana_api_url(self):
+            return super().get_europeana_api_url()
+
+    @classmethod
+    def get_info(cls):
+        return {
+            "name": "EuropeanaConfigurableSource",
+            "description": _("Configurable source for fetching photos from Europeana.com"),
+            "author": "Andrea Pasquali",
+            "version": "0.1",
+        }
+
+    def get_source_type(self):
+        return "europeana-search"
+
+    def validate(self, config):
+        try:
+            url = self.EuropeanaConfigurableDownloader(self, config).get_europeana_api_url()
+            data = Util.fetch_json(url)
+            valid = data.get("success") == True
+            return config, None if valid else _("No images found")
+        except UnsupportedConfig:
+            return config, _("Something's wrong with your search parameter")
+        except Exception as e:
+            if isinstance(e, HTTPError) and e.response.status_code == 404:
+                return config, _("No images found")
+            else:
+                return config, _("Oops, this didn't work. Is the remote service up?")
+
+    def create_downloader(self, config):
+        return self.EuropeanaConfigurableDownloader(self, config)
+
+    def get_ui_instruction(self):
+        return _(
+            "We use the <a href='https://europeana.eu'>Europeana</a> API to fetch random artwork images that match the given search keyword.\n"
+            "The Europeana API is rate-limited, so Variety fetches images from Europeana sources as a reduced rate.\n"
+            "\n"
+            "Please specify a search keyword\n"
+            "Keyword example: sea, forest, Rome"
+        )
+
+    def get_ui_short_instruction(self):
+        return _("Search keyword: ")
+
+    def get_ui_short_description(self):
+        return _("Fetch artwork images from Europeana.eu for a given keyword")
+
+    def get_source_name(self):
+        return "Europeana"
+
+    def get_default_throttling(self):
+        return Throttling(max_downloads_per_hour=120, max_queue_fills_per_hour=4)

variety/plugins/builtin/downloaders/EuropeanaDownloader.py

@@ -0,0 +1,126 @@
+# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
+### BEGIN LICENSE
+# Copyright (c) 2012, Peter Levi <peterlevi@peterlevi.com>
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+### END LICENSE
+import logging
+import random
+
+from variety.plugins.downloaders.ImageSource import Throttling
+from variety.plugins.downloaders.SimpleDownloader import SimpleDownloader
+from variety.Util import Util, _
+
+logger = logging.getLogger("variety")
+
+random.seed()
+
+
+class EuropeanaDownloader(SimpleDownloader):
+
+    DESCRIPTION = _("Europe's digital cultural heritage from Europeana.eu")
+
+    # This API key has been requested from Andrea Pasquali to Europeana.ue in order to use it for Variety project
+    API_KEY = "damolystist"
+
+    @classmethod
+    def get_info(cls):
+        return {
+            "name": "EuropeanaDownloader",
+            "description": EuropeanaDownloader.DESCRIPTION,
+            "author": "Andrea Pasquali",
+            "version": "0.1",
+        }
+
+    def get_source_type(self):
+        return "Europeana"
+
+    def get_description(self):
+        return EuropeanaDownloader.DESCRIPTION
+
+    def get_source_name(self):
+        return "Europeana.eu"
+
+    def get_source_location(self):
+        return "https://www.europeana.eu"
+
+    def get_folder_name(self):
+        return "Europeana"
+
+    def get_default_throttling(self):
+        return Throttling(max_downloads_per_hour=120, max_queue_fills_per_hour=4)
+
+    def get_europeana_api_url(self):
+        url = "https://api.europeana.eu/record/v2/search.json?wskey={api_key}&query={query}&sort={sort}&rows={rows}&profile={profile}&reusability={reusability}&media={media}&europeana_completeness:{completeness}&qf=collection:{collection}&qf=TYPE:{type}&qf=IMAGE_COLOUR:{img_color}&qf=IMAGE_SIZE:{img_size}&qf=IMAGE_ASPECTRATIO:{img_ratio}&qf=MIME_TYPE:{mime}"
+        return url.format(
+            api_key=EuropeanaDownloader.API_KEY,
+            query=f"{self.config if self.config else ''}(painting OR watercolor OR canvas OR artwork) NOT photograph NOT manuscript NOT print NOT book",
+            sort="random",
+            rows=30,
+            profile="minimal",
+            reusability="open",
+            media="true",
+            completeness="[1 TO 10]",
+            collection="art",
+            type="IMAGE",
+            img_color="true",
+            img_size="extra_large",
+            img_ratio="landscape",
+            mime="image/jpeg",
+        )
+
+    def fill_queue(self):
+
+        url = self.get_europeana_api_url()
+        logger.info(lambda: "Filling Europeana queue from " + url)
+
+        r = Util.request(url)
+        queue = []
+        for item in r.json()["items"]:
+            try:
+
+                image_url = item["edmIsShownBy"][0]
+                origin_url = item["guid"]
+
+                author = [
+                    creator
+                    for creator in (
+                        item["dcCreator"] if item.get("dcCreator", None) else []
+                    )
+                    if "http" not in creator
+                ]
+                author = author[0] if len(author) > 0 else item["dataProvider"][0]
+                author_url = [
+                    creator
+                    for creator in (item["dcCreator"] if item.get("dcCreator", None) else [])
+                    if "http" in creator
+                ]
+                artwork_url = item['edmIsShownAt'][0] if item.get('edmIsShownAt') else origin_url
+                extra_metadata = {
+                    "sourceType": "europeana",
+                    "headline": item["title"][-1],
+                    "author": author,
+                    "authorURL": author_url[0] if len(author_url) > 0 else artwork_url,
+                    "description": item["dcDescription"][0][:10000] if item.get("dcDescription") else None,
+                    "keywords": [],
+                    "extraData": {
+                        "provider": item["dataProvider"][0],
+                    },
+                }
+
+                queue.append((origin_url, image_url, extra_metadata))
+            except:
+                logger.exception(lambda: "Could not process an item from Europeana")
+                raise
+
+        random.shuffle(queue)
+        return queue