descriptions networks

Author: dale-wahl

processors/networks/clarifai_bipartite_network.py

@@ -1,7 +1,7 @@
 """
 Google Vision API co-label network
 """
-from backend.lib.processor import BasicProcessor
+from backend.lib.processor import BasicProcessor, ProcessorDescription
 from common.lib.compatibility import Compatibility
 from common.lib.outputs import Network
 from common.lib.helpers import UserInput
@@ -19,15 +19,16 @@ class VisionTagBiPartiteNetworker(BasicProcessor):
     Google Vision API co-label network
     """
     type = "clarifai-bipartite-network"  # job type ID
-    category = "Networks"  # category
-    title = "Clarifai Bipartite Annotation Network"  # title displayed in UI
-    description = "Create a GEXF network file comprised of all annotations returned by the Clarifai API. Labels " \
-                  "returned by the API, and image file names, are nodes. Edges are created between file names and " \
-                  "labels if the label occurs for the image with that file name."
+    description = ProcessorDescription(
+        title="Create Clarifai co-label network",
+        category="Networks",
+        tags=["network"],
+        description="Build a bipartite network from Clarifai image annotations. Image file names and Clarifai labels are the nodes, and an edge connects a file name to a label when Clarifai assigned that label to that image. Use the minimum confidence option to drop low-confidence labels.",
+        icon="circle-nodes",
+    )
     extension = "gexf"  # extension of result file, used internally and in UI
     # a graph file, no column table
     output = Network()
-    icon = "circle-nodes"
 
     # Allow processor to run on Clarifai API data
     compatibility = Compatibility(types={"clarifai-api"})

processors/networks/colink_urls.py

@@ -7,7 +7,7 @@
 
 import psutil
 
-from backend.lib.processor import BasicProcessor
+from backend.lib.processor import BasicProcessor, ProcessorDescription
 from common.lib.compatibility import Compatibility
 from common.lib.outputs import Network
 from common.lib.exceptions import ProcessorInterruptedException
@@ -29,14 +29,16 @@ class URLCoLinker(BasicProcessor):
 	Generate URL co-link network
 	"""
 	type = "url-network"  # job type ID
-	category = "Networks"  # category
-	title = "URL co-occurence network"  # title displayed in UI
-	description = "Create a GEXF network file comprised of URLs appearing together (in a post or thread). " \
-				  "Edges are weighted by amount of co-links."  # description displayed in UI
+	description = ProcessorDescription(
+		title="Create URL co-link network",
+		category="Networks",
+		tags=["urls", "network"],
+		description="Build a network of URLs that appear together in the same post or thread. Each URL is a node, and edges connect URLs that co-occur. Choose whether to use full URLs or only domain names, and whether to count co-occurrence per post or per thread.",
+		icon="circle-nodes",
+	)
 	extension = "gexf"  # extension of result file, used internally and in UI
 	# a graph file, no column table
 	output = Network()
-	icon = "circle-nodes"
 
 	# Allow on top-level CSV/NDJSON datasets
 	compatibility = Compatibility(top_dataset_only=True, extensions={"csv", "ndjson"})

processors/networks/cotag_network.py

@@ -3,6 +3,7 @@
 """
 
 from backend.lib.preset import ProcessorPreset
+from backend.lib.processor import ProcessorDescription
 from common.lib.helpers import UserInput
 from common.lib.compatibility import Compatibility
 from common.lib.outputs import Delegated
@@ -18,15 +19,16 @@ class CoTaggerPreset(ProcessorPreset):
     Generate co-tag network of co-occurring (hash)tags in items
     """
     type = "preset-cotag-network"  # job type ID
-    category = "Networks"  # category
-    title = "Co-tag network"  # title displayed in UI
-    description = "Create a GEXF network file of tags co-occurring in a posts. " \
-                  "Edges are weighted by the amount of tag co-occurrences; nodes " \
-                  "are weighted by how often the tag appears in the dataset."  # description displayed in UI
+    description = ProcessorDescription(
+        title="Co-tag network",
+        category="Networks",
+        tags=["hashtags", "network"],
+        description="Create a network of tags co-occurring in items. Edges are weighted by how often two tags co-occur; nodes are weighted by how often a tag appears in the dataset.",
+        icon="circle-nodes",
+    )
     extension = "gexf"  # extension of result file, used internally and in UI
     # a preset; its output is its last step's
     output = Delegated()
-    icon = "circle-nodes"
 
 
     possible_tag_columns = {"tags", "hashtags", "groups"}

processors/networks/coword_network.py

@@ -3,6 +3,7 @@
 """
 
 from backend.lib.preset import ProcessorPreset
+from backend.lib.processor import ProcessorDescription
 from common.lib.compatibility import Compatibility
 from common.lib.outputs import Delegated
 
@@ -17,13 +18,14 @@ class CowordNetworker(ProcessorPreset):
     Generate co-word network
     """
     type = "preset-coword-network"  # job type ID
-    category = "Networks"  # category
-    title = "Co-word network"  # title displayed in UI
-    description = "Create a GEXF network file of word co-occurences. Edges denote " \
-                  "words that appear close to each other. Edges and nodes are weighted by the " \
-                  "amount of co-word occurrences."  # description displayed in UI
+    description = ProcessorDescription(
+        title="Co-word network",
+        category="Networks",
+        tags=["network"],
+        description="Create a network of word co-occurrences. Edges connect words that appear close to each other. Edges and nodes are weighted by how often the words co-occur.",
+        icon="circle-nodes",
+    )
     extension = "gexf"  # extension of result file, used internally and in UI
-    icon = "circle-nodes"
 
     # a preset; its output is its last step's
     output = Delegated()

processors/networks/gexf_to_csv.py

@@ -1,7 +1,7 @@
 """
 Convert a GEXF network file to a CSV file
 """
-from backend.lib.processor import BasicProcessor
+from backend.lib.processor import BasicProcessor, ProcessorDescription
 from common.lib.compatibility import Compatibility
 from common.lib.outputs import Table
 
@@ -20,13 +20,16 @@ class GexfToCsv(BasicProcessor):
     Convert a GEXF network file to a CSV file
     """
     type = "gexf-to-csv"
-    category = "Networks"
-    title = "Export Network as CSV Spreadsheet"
-    description = "Convert a GEXF network file to a CSV spreadsheet"
+    description = ProcessorDescription(
+        title="Convert network to CSV",
+        category="Networks",
+        tags=["convert format"],
+        description="Convert a GEXF network file to a CSV file, with one row per edge. Each row lists the source and target nodes, their attributes, and the edge attributes. Edges are sorted by weight, from most to least frequent.",
+        icon="file-csv",
+    )
     extension = "csv"
     # a derived table
     output = Table()
-    icon = "file-csv"
 
     # Allow on GEXF datasets
     compatibility = Compatibility(extensions={"gexf"})

processors/networks/google_vision_bipartite_network.py

@@ -1,7 +1,7 @@
 """
 Google Vision API co-label network
 """
-from backend.lib.processor import BasicProcessor
+from backend.lib.processor import BasicProcessor, ProcessorDescription
 from common.lib.compatibility import Compatibility
 from common.lib.outputs import Network
 from common.lib.helpers import UserInput
@@ -20,15 +20,16 @@ class VisionTagBiPartiteNetworker(BasicProcessor):
     Google Vision API co-label network
     """
     type = "vision-bipartite-network"  # job type ID
-    category = "Networks"  # category
-    title = "Google Vision bipartite annotation network"  # title displayed in UI
-    description = "Create a GEXF network file comprised of all annotations returned by the Google Vision API. Labels " \
-                  "returned by the API, and image file names, are nodes. Edges are created between file names and " \
-                  "labels if the label occurs for the image with that file name."
+    description = ProcessorDescription(
+        title="Google Vision bipartite annotation network",
+        category="Networks",
+        tags=["network"],
+        description="Create a network from annotations returned by the Google Vision API. Image file names and the labels returned for them are nodes. An edge connects a file name to a label when that label occurs for that image.",
+        icon="circle-nodes",
+    )
     extension = "gexf"  # extension of result file, used internally and in UI
     # a graph file, no column table
     output = Network()
-    icon = "circle-nodes"
 
     # Allow processor to run on Google Vision API data
     compatibility = Compatibility(types={"google-vision-api"})

processors/networks/google_vision_network.py

@@ -1,7 +1,7 @@
 """
 Google Vision API co-label network
 """
-from backend.lib.processor import BasicProcessor
+from backend.lib.processor import BasicProcessor, ProcessorDescription
 from common.lib.compatibility import Compatibility
 from common.lib.outputs import Network
 from common.lib.helpers import UserInput
@@ -20,15 +20,16 @@ class VisionTagNetworker(BasicProcessor):
     Google Vision API co-label network
     """
     type = "vision-label-network"  # job type ID
-    category = "Networks"  # category
-    title = "Google Vision co-Label network"  # title displayed in UI
-    description = "Create a GEXF network file comprised of all annotations returned by the" \
-                  "Google Vision API. Labels returned by the API are nodes. Labels occurring on the same image form" \
-                  "edges."
+    description = ProcessorDescription(
+        title="Google Vision co-label network",
+        category="Networks",
+        tags=["network"],
+        description="Create a network from annotations returned by the Google Vision API. Labels returned by the API are nodes. An edge connects two labels when they occur on the same image, weighted by how often they co-occur.",
+        icon="circle-nodes",
+    )
     extension = "gexf"  # extension of result file, used internally and in UI
     # a graph file, no column table
     output = Network()
-    icon = "circle-nodes"
 
     # Allow processor to run on Google Vision API data
     compatibility = Compatibility(types={"google-vision-api"})

processors/networks/hash_similarity_network.py

@@ -6,7 +6,7 @@
 import networkx as nx
 import numpy as np
 
-from backend.lib.processor import BasicProcessor
+from backend.lib.processor import BasicProcessor, ProcessorDescription
 from common.lib.compatibility import Compatibility
 from common.lib.outputs import Network
 from common.lib.exceptions import ProcessorException
@@ -24,13 +24,20 @@ class HashSimilarityNetworker(BasicProcessor):
     Compare hashes and generate a network based on similarity
     """
     type = "hash-similarity-network"
-    category = "Networks"
-    title = "Hash similarity network"
-    description = "Calculate similarity of hashes and create a GEXF network file. Can identify near duplicate hashes."
+    description = ProcessorDescription(
+        title="Hash similarity network",
+        category="Networks",
+        tags=["similarity", "network"],
+        description="Compare bit hashes and create a network linking similar items. Each pair of hashes is compared bit by bit, and an edge is added when they are at least as similar as the chosen threshold. Useful for finding near-duplicate images or videos.",
+        warnings=[
+            "Only bit hashes are supported, such as those produced by the video hasher.",
+            "Every pair of hashes is compared, so this can be slow on large datasets.",
+        ],
+        icon="circle-nodes",
+    )
     extension = "gexf"
     # a graph file, no column table
     output = Network()
-    icon = "circle-nodes"
 
     # Runs on the video hasher's output (like the other video-hash networks); any
     # dataset of bit-hash rows would work.

processors/networks/image-network.py

@@ -3,7 +3,7 @@
 """
 import json
 
-from backend.lib.processor import BasicProcessor
+from backend.lib.processor import BasicProcessor, ProcessorDescription
 from common.lib.helpers import hash_file
 
 import networkx as nx
@@ -27,16 +27,19 @@ class ImageGrapher(BasicProcessor):
     images were sourced from
     """
     type = "image-bipartite-network"  # job type ID
-    category = "Networks"
-    title = "Bipartite image-item network"  # title displayed in UI
-    description = ("Create a GEXF network file with a bipartite network of "
-                   "images and some data field (e.g. author) of the dataset "
-                   "the images were sourced from. Suitable for use with Gephi's "
-                   "'Image Preview' plugin.")
+    description = ProcessorDescription(
+        title="Bipartite image-item network",
+        category="Networks",
+        tags=["network", "authors"],
+        description="Create a network with a bipartite network of images and a data field (for example author) of the dataset the images were sourced from. Suitable for use with Gephi's Image Preview plugin.",
+        info=[
+            "Optionally merge similar images into a single node using file hashing or perceptual hashing.",
+        ],
+        icon="circle-nodes",
+    )
     extension = "gexf"  # extension of result file, used internally and in UI
     # a graph file, no column table
     output = Network()
-    icon = "circle-nodes"
 
     # coarse map spec; is_compatible_with (below) is the runtime truth -- it also walks the
     # genealogy to find an image-downloader root (get_root_dataset)

processors/networks/quote_network.py

@@ -3,7 +3,7 @@
 """
 import re
 
-from backend.lib.processor import BasicProcessor
+from backend.lib.processor import BasicProcessor, ProcessorDescription
 from common.lib.compatibility import Compatibility
 from common.lib.outputs import Network
 
@@ -21,14 +21,16 @@ class QuoteNetworkGrapher(BasicProcessor):
 	Creates a network of posts quoting each other
 	"""
 	type = "quote-network"  # job type ID
-	category = "Networks"
-	title = "Reply network"  # title displayed in UI
-	description = "Create a GEXF network file of posts replying to each other. " \
-				  "Each reference to another post creates an edge between posts. "  # description displayed in UI
+	description = ProcessorDescription(
+		title="Reply network",
+		category="Networks",
+		tags=["network"],
+		description="Create a network of posts replying to each other. Each reference to another post creates an edge between the two posts.",
+		icon="circle-nodes",
+	)
 	extension = "gexf"  # extension of result file, used internally and in UI
 	# a graph file, no column table
 	output = Network()
-	icon = "circle-nodes"
 
 	# chan datasets (posts reply to / quote each other)
 	compatibility = Compatibility(datasources={"fourchan", "eightchan", "eightkun"})