Merge pull request #27 from feedzai/fix-model-with-non-thread-safe-dependencies

Fix load of models that depend on non thread-safe dependencies

Author: paulojrp

openml-python-common/README.md

@@ -40,3 +40,15 @@ export PATH=$ANACONDA_PATH/envs/myenv/bin:$PATH
 export LD_LIBRARY_PATH=$ANACONDA_PATH/envs/myenv/lib/python3.6/site-packages/jep:$LD_LIBRARY_PATH
 export LD_PRELOAD=$ANACONDA_PATH/envs/myenv/lib/libpython3.6m.so
 ```
+
+7. If you need to share Python modules across sub-interpreters, you will need to create a "python-packages.xml" file where you define the modules to be shared. By default the provider is already sharing the "numpy" and "tensorflow" modules. This is a workaround for the issues with CPython extensions.
+    - Remember that this file should be added to the classpath of your program.
+
+```
+<?xml version="1.0"?>
+<python>
+   <package>my_package_1</package>
+   <package>my_package_2</package>
+</python>
+
+``` 

openml-python-common/pom.xml

@@ -42,16 +42,16 @@
             <groupId>com.feedzai</groupId>
             <artifactId>openml-utils</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>com.feedzai</groupId>
             <artifactId>openml-utils</artifactId>
             <scope>test</scope>
             <type>test-jar</type>
         </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <scope>provided</scope>
-        </dependency>
     </dependencies>
 </project>

openml-python-common/src/main/java/com/feedzai/openml/python/jep/instance/JepInstance.java

@@ -15,12 +15,16 @@
  */
 package com.feedzai.openml.python.jep.instance;
 
+import com.feedzai.openml.python.modules.SharedModulesParser;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.util.concurrent.Uninterruptibles;
 import jep.Jep;
+import jep.JepConfig;
 import jep.JepException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
@@ -97,8 +101,18 @@ public void stop() {
      */
     @Override
     public void run() {
-
-        try (final Jep jep = new Jep(false)) {
+        final Set<String> sharedModules = ImmutableSet.<String>builder()
+                .add("tensorflow")
+                .add("numpy")
+                .addAll(new SharedModulesParser().getSharedModules())
+                .build();
+        logger.debug("Python modules to be shared: {}", String.join(",", sharedModules.toString()));
+
+        final JepConfig jepConfig = new JepConfig()
+                .addSharedModules(sharedModules.toArray(new String[0]))
+                .setInteractive(false);
+
+        try (final Jep jep = new Jep(jepConfig)) {
             while (this.running) {
                 this.evaluationQueue.take().evaluate(jep);
             }

openml-python-common/src/main/java/com/feedzai/openml/python/modules/SharedModulesParser.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2018 Feedzai
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.feedzai.openml.python.modules;
+
+import com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.InputStream;
+import java.util.Set;
+
+/**
+ * Class responsible for parsing a XML file in order to retrieve the name of the Python modules to be shared across
+ * sub-interpreters.
+ *
+ * @author Paulo Pereira (paulo.pereira@feedzai.com)
+ * @since 0.1.5
+ */
+public class SharedModulesParser {
+
+    /**
+     * Logger.
+     */
+    private static final Logger logger = LoggerFactory.getLogger(SharedModulesParser.class);
+
+    /**
+     * Default value for {@link #xmlFile}.
+     */
+    private static final String DEFAULT_XML_FILE = "python-packages.xml";
+
+    /**
+     * The filename of a XML file with the name of the Python modules to be shared across sub-interpreters.
+     */
+    private final String xmlFile;
+
+    /**
+     * Constructor.
+     *
+     * @param xmlFileName The name of a XML file.
+     */
+    public SharedModulesParser(final String xmlFileName) {
+        this.xmlFile = xmlFileName;
+    }
+
+    /**
+     * Constructor.
+     */
+    public SharedModulesParser() {
+        this(DEFAULT_XML_FILE);
+    }
+
+    /**
+     * Gets the {@link InputStream} of {@link #xmlFile} that exists in the current classpath.
+     *
+     * @return The {@link InputStream} of {@link #xmlFile}.
+     */
+    private InputStream getXMLInputStream() {
+        final ClassLoader classLoader = getClass().getClassLoader();
+        return classLoader.getResourceAsStream(this.xmlFile);
+    }
+
+    /**
+     * Parses the {@link #xmlFile} to retrieve the name of the Python modules to be shared across sub-interpreters.
+     *
+     * @return A {@link Set} with the name of the Python modules to be shared across sub-interpreters.
+     * @throws Exception If there is an error while parsing {@link #xmlFile}.
+     */
+    private Set<String> parseXMLFile() throws Exception {
+        final ImmutableSet.Builder<String> sharedModulesBuilder = ImmutableSet.builder();
+        final DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+        final DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
+
+        try (final InputStream xmlFile = getXMLInputStream()) {
+            final Document doc = dBuilder.parse(xmlFile);
+            doc.getDocumentElement().normalize();
+
+            final NodeList nList = doc.getElementsByTagName("package");
+            for (int i = 0; i < nList.getLength(); i++) {
+                sharedModulesBuilder.add(nList.item(i).getFirstChild().getNodeValue());
+            }
+        }
+        return sharedModulesBuilder.build();
+    }
+
+    /**
+     * Retrieves a {@link Set} with the name of the Python modules to be shared across sub-interpreters.
+     *
+     * @return The name of the Python modules to be shared across sub-interpreters.
+     */
+    public Set<String> getSharedModules() {
+        Set<String> sharedModules = ImmutableSet.of();
+        try {
+            sharedModules = parseXMLFile();
+        } catch (final Exception e) {
+            logger.warn("Problem while getting the XML file with the Python modules to be shared.", e);
+        }
+        return sharedModules;
+    }
+}

openml-python-common/src/main/java/com/feedzai/openml/python/modules/package-info.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2018 Feedzai
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/**
+ * This package contains logic for parsing XML files to retrieve the name of the Python modules to be shared across
+ * sub-interpreters.
+ *
+ * @since 0.1.5
+ */
+package com.feedzai.openml.python.modules;

openml-python-common/src/test/java/com/feedzai/openml/python/jep/instance/JepInstanceTest.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2018 Feedzai
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.feedzai.openml.python.jep.instance;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Contains the tests for the wrapper for the Jep object.
+ *
+ * @author Paulo Pereira (paulo.pereira@feedzai.com)
+ * @since 0.1.5
+ */
+public class JepInstanceTest {
+
+    /**
+     * The wrapper for the Jep object used in the tests.
+     */
+    private JepInstance jepInstance;
+
+    /**
+     * Initializes an instance of {@link JepInstance}.
+     */
+    @Before
+    public void setUp() {
+        this.jepInstance = new JepInstance();
+        this.jepInstance.start();
+    }
+
+    /**
+     * Tears downs the instance of {@link JepInstance}.
+     */
+    @After
+    public void tearDown() {
+        this.jepInstance.stop();
+    }
+
+    /**
+     * Tests the submission of evaluations on a {@link JepInstance}.
+     *
+     * @throws Exception If there is a problem while getting the result.
+     */
+    @Test
+    public void submitEvaluationTest() throws Exception {
+        final String result = this.jepInstance.submitEvaluation((jep) -> jep.getValue("1 + 2")).get().toString();
+        assertThat(result)
+            .as("The result of the evaluation")
+            .isEqualTo("3");
+    }
+}

openml-python-common/src/test/java/com/feedzai/openml/python/jep/instance/package-info.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2018 Feedzai
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/**
+ * This package contains the unit-tests for {@link com.feedzai.openml.python.jep.instance}.
+ *
+ * @since 0.1.5
+ */
+package com.feedzai.openml.python.jep.instance;

openml-python-common/src/test/java/com/feedzai/openml/python/modules/SharedModulesParserTest.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2018 Feedzai
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.feedzai.openml.python.modules;
+
+import org.assertj.core.util.Files;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.Set;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests the retrieving of the modules to be shared across sub-interpreters from a XML file.
+ *
+ * @author Paulo Pereira (paulo.pereira@feedzai.com)
+ * @since 0.1.5
+ */
+public class SharedModulesParserTest {
+
+    /**
+     * Tests the retrieving of the shared modules from a valid XML file with name of two modules to be shared.
+     */
+    @Test
+    public void validXMLFileTest() {
+        final Set<String> sharedPythonPackages = new SharedModulesParser().getSharedModules();
+        assertThat(sharedPythonPackages)
+                .as("Set of shared modules.")
+                .hasSize(2)
+                .contains("my_package_1", "my_package_2");
+    }
+
+    /**
+     * Tests the retrieving of the shared modules from an empty file.
+     */
+    @Test
+    public void emptyFileTest() {
+        final File file = Files.newTemporaryFile();
+        file.deleteOnExit();
+
+        final SharedModulesParser sharedModulesParser = new SharedModulesParser(file.getAbsolutePath());
+        assertThat(sharedModulesParser.getSharedModules())
+                .as("Set of shared modules.")
+                .hasSize(0);
+    }
+
+    /**
+     * Tests the retrieving of the shared modules from a non existing file.
+     */
+    @Test
+    public void nonExistingFileTest() {
+        final SharedModulesParser sharedModulesParser = new SharedModulesParser("non_existing_file");
+        assertThat(sharedModulesParser.getSharedModules())
+                .as("Set of shared modules.")
+                .hasSize(0);
+    }
+}