[pythonscripting] Add scope typings and update helper lib to version 1.0.20 (#21214)

Signed-off-by: Holger Hees <holger.hees@gmail.com>

Author: HolgerHees

bundles/org.openhab.automation.pythonscripting/README.md

@@ -220,6 +220,18 @@ As a final step, the folders `/openhab/conf/automation/python/libs/` and `/openh
 
 ![Pythonscripting autocompletion](doc/ide_autocompletion.png)
 
+#### Autocompletion on dynamic results with custom type hints
+
+Typings are not working on methods with an dynamic result types like `Registry.getItemState`. To force a specific expected result type, you can annotate the variable like below.
+
+```python
+from org.openhab.core.library.types import DecimalType
+
+item: DecimalType = Registry.getItemState("TestNumberItem")
+```
+
+![Pythonscripting autocompletion](doc/ide_autocompletion_custom.png)
+
 ## Typical log errors
 
 ### Graal python language not initialized. ...
@@ -279,4 +291,10 @@ You should also check your logs for a message related to the helper lib deployme
 
 ## Limitations
 
-- GraalPy can't handle arguments in constructors of Java objects. Means you can't instantiate a Java object in Python with a parameter. <https://github.com/oracle/graalpython/issues/367>
+### Avoid using datetime.strptime
+
+You should avoid using `datetime.strptime` because there is a bug in GraalPy associated with it. Once this function is used, it can change the JVM-wide default time zone for the whole openHAB process until it is reset or openHAB is restarted.
+
+### Constructors of Java objects
+
+GraalPy can't handle arguments in constructors of Java objects. Means you can't instantiate a Java object in Python with a parameter. <https://github.com/oracle/graalpython/issues/367>

bundles/org.openhab.automation.pythonscripting/doc/ide_autocompletion_custom.png

@@ -15,7 +15,7 @@
   <name>openHAB Add-ons :: Bundles :: Automation :: Python Scripting</name>
 
   <properties>
-    <helperlib.version>1.0.19</helperlib.version>
+    <helperlib.version>1.0.20</helperlib.version>
   </properties>
 
   <dependencies>

bundles/org.openhab.automation.pythonscripting/pom.xml

@@ -29,6 +29,7 @@
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
@@ -291,7 +292,7 @@ public void accept(Path path) {
                 getBindings(ScriptContext.ENGINE_SCOPE).put(ScriptExtensionModuleProvider.IMPORT_PROXY_NAME,
                         wrapImportFn);
                 try {
-                    if (!isScriptFile() && !isScriptModule() && !isTransformation()) {
+                    if (!isScriptFile() && !isScriptModule() && !isTransformation() && !isCLI()) {
                         logger.warn(
                                 "Unknown script environment detected for engine '{}': Neither script file, script module nor transformation.",
                                 engineIdentifier);
@@ -540,6 +541,20 @@ private boolean isTransformation() {
         return engineIdentifier.startsWith(OPENHAB_TRANSFORMATION_SCRIPT);
     }
 
+    /**
+     * Tests if the script is used in the karaf console, i.e. created from the ConsoleCommandExtension service.
+     *
+     * @return true if it is a cli script, false otherwise
+     */
+    private boolean isCLI() {
+        ScriptContext ctx = getContext();
+        if (ctx == null) {
+            logger.warn("Failed to retrieve script context from engine '{}'.", engineIdentifier);
+            return false;
+        }
+        return this.engineIdentifier.startsWith("pythonscripting-cli");
+    }
+
     private static Set<String> transformArrayToSet(Value value) {
         try {
             Set<String> set = new HashSet<>();
@@ -590,4 +605,8 @@ private static ZonedDateTime parseDatetime(Value value) {
                         ? OffsetDateTime.now().getOffset().getId()
                         : ""));
     }
+
+    public Map<String, Object> getScope() {
+        return scriptExtensionModuleProvider.getScope();
+    }
 }

bundles/org.openhab.automation.pythonscripting/src/main/java/org/openhab/automation/pythonscripting/internal/PythonScriptEngine.java

@@ -229,7 +229,7 @@ private void executeTyping(Console console) {
                     + PythonScriptEngineConfiguration.PYTHON_TYPINGS_PATH + "'.")) {
                 return;
             }
-            new TypingCmd(new TypingCmd.Logger(console)).build();
+            new TypingCmd(new TypingCmd.Logger(console), scriptEngineManager).build();
         } catch (Exception e) {
             throw new IllegalArgumentException(e);
         }
@@ -286,7 +286,7 @@ private void printLoadingMessage(Console console, boolean show) {
      * including any injected required modules.
      */
     private @Nullable Object executePython(Console console, EngineEvalFunction process, boolean withFullContext) {
-        String scriptIdentifier = "python-console-" + UUID.randomUUID().toString();
+        String scriptIdentifier = "pythonscripting-cli-" + UUID.randomUUID().toString();
         ScriptEngine engine = null;
 
         try {

bundles/org.openhab.automation.pythonscripting/src/main/java/org/openhab/automation/pythonscripting/internal/console/PythonConsoleCommandExtension.java

@@ -19,19 +19,28 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
+import java.util.UUID;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import javax.script.ScriptEngine;
+
 import org.eclipse.jdt.annotation.NonNullByDefault;
+import org.openhab.automation.pythonscripting.internal.PythonScriptEngine;
 import org.openhab.automation.pythonscripting.internal.PythonScriptEngineConfiguration;
+import org.openhab.automation.pythonscripting.internal.PythonScriptEngineFactory;
 import org.openhab.automation.pythonscripting.internal.console.handler.typing.ClassCollector;
 import org.openhab.automation.pythonscripting.internal.console.handler.typing.ClassCollector.ClassContainer;
 import org.openhab.automation.pythonscripting.internal.console.handler.typing.ClassConverter;
+import org.openhab.core.automation.module.script.ScriptEngineContainer;
+import org.openhab.core.automation.module.script.ScriptEngineManager;
 import org.openhab.core.io.console.Console;
 
 /**
@@ -42,11 +51,14 @@
 @NonNullByDefault
 public class TypingCmd {
     private final Logger logger;
+    private final ScriptEngineManager scriptEngineManager;
 
+    private static final String OPENHAB_PACKAGE_PREFIX = "org.openhab";
     private static final String PATH_SEPARATOR = FileSystems.getDefault().getSeparator();
 
-    public TypingCmd(Logger logger) {
+    public TypingCmd(Logger logger, ScriptEngineManager scriptEngineManager) {
         this.logger = logger;
+        this.scriptEngineManager = scriptEngineManager;
     }
 
     public void build() throws Exception {
@@ -62,17 +74,28 @@ public void build() throws Exception {
 
         Map<String, ClassContainer> fileContainerMap = new HashMap<String, ClassContainer>();
         Set<String> imports = new HashSet<String>();
-        // Collect Bundle Classes
-        Map<String, ClassContainer> bundleClassMap = collector.collectBundleClasses("org.openhab");
+        Set<String> dumped = new HashSet<String>();
+
+        // 1. The bundle collector is collecting all public openhab classes and all used/imported classes
+        Map<String, ClassContainer> bundleClassMap = collector.collectBundleClasses(OPENHAB_PACKAGE_PREFIX);
+
+        // 2. The scope object is dumped
+        // All referenced openhab classes which are not already collected, are "registered" in the bundleClassMap
+        // And all used/imported class are added to the import list
+        Collection<String> scopeImports = dumpScope(outputPath, bundleClassMap);
+        imports.addAll(scopeImports);
+
+        // 3. All openhab classes are dumped
         for (ClassContainer container : bundleClassMap.values()) {
             ClassConverter converter = new ClassConverter(container);
             String classBody = converter.build();
             imports.addAll(converter.getImports());
             dumpClassContentToFile(classBody, container, outputPath, fileContainerMap);
+            dumped.add(container.getRelatedClass().getName());
         }
 
-        imports = imports.stream().filter(i -> !i.startsWith("org.openhab")).collect(Collectors.toSet());
-
+        // 4. All collected imports are dumped, if they are not already dumped before
+        imports = imports.stream().filter(i -> !dumped.contains(i)).collect(Collectors.toSet());
         Map<String, ClassContainer> reflectionClassMap = collector.collectReflectionClasses(imports);
         for (ClassContainer container : reflectionClassMap.values()) {
             ClassConverter converter = new ClassConverter(container);
@@ -88,6 +111,86 @@ public void build() throws Exception {
                 + outputPath + "'");
     }
 
+    private Collection<String> dumpScope(Path outputPath, Map<String, ClassContainer> bundleClassMap)
+            throws IOException {
+        Map<String, String> imports = new HashMap<String, String>();
+        String identifier = "pythonscripting-cli-" + UUID.randomUUID().toString();
+        try {
+            ScriptEngineContainer container = scriptEngineManager
+                    .createScriptEngine(PythonScriptEngineFactory.SCRIPT_TYPE, identifier);
+            if (container != null) {
+                StringBuilder scopeBody = new StringBuilder();
+                ScriptEngine engine = container.getScriptEngine();
+
+                Map<String, Object> scope = ((PythonScriptEngine) engine).getScope();
+                for (Entry<String, Object> entry : scope.entrySet()) {
+                    Object value = entry.getValue();
+                    Class cls;
+                    String packageName;
+                    String pythonClassName;
+                    String pythonModuleName;
+                    String definition;
+
+                    if (value instanceof Class) {
+                        cls = (Class) value;
+                        packageName = cls.getName();
+
+                        pythonClassName = ClassContainer.parsePythonClassName(packageName);
+                        pythonModuleName = ClassContainer.parsePythonModuleName(packageName);
+                        definition = entry.getKey() + ": Type[_" + pythonClassName + "] = _" + pythonClassName;
+                    } else {
+                        cls = value.getClass();
+                        packageName = value.getClass().getName();
+
+                        if ("org.openhab.automation.pythonscripting.internal.provider.LifecycleTracker"
+                                .equals(packageName)) {
+                            packageName = "org.openhab.core.automation.module.script.LifecycleScriptExtensionProvider$LifecycleTracker";
+                        } else if (packageName.endsWith("Impl") || packageName.endsWith("Delegate")) {
+                            cls = value.getClass().getInterfaces()[0];
+                            packageName = cls.getName();
+                        }
+
+                        pythonClassName = ClassContainer.parsePythonClassName(packageName);
+                        pythonModuleName = ClassContainer.parsePythonModuleName(packageName);
+
+                        if (cls.isEnum()) {
+                            definition = entry.getKey() + ": _" + pythonClassName + " = " + "_" + pythonClassName + "."
+                                    + entry.getKey();
+                        } else {
+                            definition = entry.getKey() + ": _" + pythonClassName;
+                        }
+                    }
+
+                    if (packageName.startsWith(OPENHAB_PACKAGE_PREFIX) && !bundleClassMap.containsKey(packageName)) {
+                        bundleClassMap.put(packageName, new ClassContainer(cls));
+                    }
+
+                    imports.put(packageName,
+                            "from " + pythonModuleName + " import " + pythonClassName + " as _" + pythonClassName);
+
+                    String classUrl = ClassConverter.buildDocumentationLink(packageName);
+
+                    scopeBody.append(definition);
+                    scopeBody.append("\n");
+                    scopeBody.append("\"\"\"\n");
+                    scopeBody.append("Java class: ").append(packageName).append("\n\n");
+                    scopeBody.append("Java doc: ").append(classUrl).append("\n");
+                    scopeBody.append("\"\"\"\n\n");
+                }
+
+                scopeBody.insert(0, "\n\n");
+                scopeBody.insert(0, "from typing import Type");
+                scopeBody.insert(0, ClassConverter.buildClassImports(imports.values()));
+
+                dumpContentToFile(scopeBody.toString(), outputPath.resolve("scope.py"));
+            }
+        } finally {
+            scriptEngineManager.removeEngine(identifier);
+        }
+
+        return imports.keySet();
+    }
+
     public void dumpInit(String path, Map<String, ClassContainer> fileContainerMap) throws IOException {
         File root = new File(path);
         File[] list = root.listFiles();
@@ -105,7 +208,7 @@ public void dumpInit(String path, Map<String, ClassContainer> fileContainerMap)
                 StringBuilder initBody = new StringBuilder();
                 // List<String> modules = new ArrayList<String>();
                 for (File file : files) {
-                    if (file.toString().endsWith("__init__.py")) {
+                    if (file.toString().endsWith("__init__.py") || file.toString().endsWith("scope.py")) {
                         continue;
                     }
                     ClassContainer container = fileContainerMap.get(file.toString());

bundles/org.openhab.automation.pythonscripting/src/main/java/org/openhab/automation/pythonscripting/internal/console/handler/TypingCmd.java

@@ -112,7 +112,7 @@ public int compare(MethodContainer o1, MethodContainer o2) {
         // Class imports
         if (!imports.isEmpty()) {
             classBody.insert(0, "\n\n");
-            classBody.insert(0, buildClassImports());
+            classBody.insert(0, buildClassImports(imports.values()));
         }
 
         return classBody.toString();
@@ -263,28 +263,6 @@ private String buildClassMethod(MethodContainer method) {
         return builder.toString();
     }
 
-    private Object buildClassImports() {
-        StringBuilder builder = new StringBuilder();
-        HashSet<String> hashSet = new HashSet<>(imports.values());
-        ArrayList<String> sortedImports = new ArrayList<>(hashSet);
-        Collections.sort(sortedImports, new Comparator<String>() {
-            @Override
-            public int compare(String o1, String o2) {
-                if (o1.length() > o2.length()) {
-                    return 1;
-                }
-                if (o1.length() < o2.length()) {
-                    return -1;
-                }
-                return o1.compareTo(o2);
-            }
-        });
-        for (String importLine : sortedImports) {
-            builder.append(importLine + "\n");
-        }
-        return builder.toString();
-    }
-
     private JavaType collectJavaTypes(Type genericType, Map<String, String> generics) {
         JavaType javaType = null;
         if (genericType instanceof TypeVariable) {
@@ -509,8 +487,7 @@ private String cleanClassName(String className) {
             return null;
         }
 
-        String classUrl = baseUrl
-                + container.getRelatedClass().getName().toLowerCase().replace(".", "/").replace("$", ".");
+        String classUrl = buildDocumentationLink(container.getRelatedClass().getName());
 
         StringBuilder builder = new StringBuilder();
         builder.append("    \"\"\"\n");
@@ -526,8 +503,7 @@ private String cleanClassName(String className) {
             return null;
         }
 
-        String classUrl = baseUrl
-                + container.getRelatedClass().getName().toLowerCase().replace(".", "/").replace("$", ".");
+        String classUrl = buildDocumentationLink(container.getRelatedClass().getName());
 
         StringBuilder builder = new StringBuilder();
         builder.append("        \"\"\"\n");
@@ -548,6 +524,32 @@ private String cleanClassName(String className) {
         return builder.toString();
     }
 
+    public static String buildDocumentationLink(String classname) {
+        return baseUrl + classname.toLowerCase().replace(".", "/").replace("$", ".");
+    }
+
+    public static String buildClassImports(Collection<String> imports) {
+        StringBuilder builder = new StringBuilder();
+        Set<String> hashSet = new HashSet<>(imports);
+        List<String> sortedImports = new ArrayList<>(hashSet);
+        Collections.sort(sortedImports, new Comparator<String>() {
+            @Override
+            public int compare(String o1, String o2) {
+                if (o1.length() > o2.length()) {
+                    return 1;
+                }
+                if (o1.length() < o2.length()) {
+                    return -1;
+                }
+                return o1.compareTo(o2);
+            }
+        });
+        for (String importLine : sortedImports) {
+            builder.append(importLine + "\n");
+        }
+        return builder.toString();
+    }
+
     public static class JavaType {
         private final String type;
         private final List<JavaType> subTypes = new ArrayList<JavaType>();

bundles/org.openhab.automation.pythonscripting/src/main/java/org/openhab/automation/pythonscripting/internal/console/handler/typing/ClassConverter.java

@@ -96,6 +96,10 @@ public void put(String key, Object value) {
         this.globals.put(key, value);
     }
 
+    public Map<String, Object> getScope() {
+        return new HashMap<>(this.globals);
+    }
+
     public static interface ModuleLocator {
         Map<String, Object> locateModule(String name, List<String> fromlist);
     }