NPM integration first implementation: downloads dependencies into a target folder and collect them appending version to the folder name

Author: Federico Fissore

app/lib/jackson-core-lgpl-1.9.12.jar

@@ -0,0 +1,10 @@
+package processing.app.packages;
+
+import java.io.File;
+import java.io.IOException;
+
+public interface DependenciesResolver {
+
+  void resolveAndDownload(File libFolder, File targetFolder) throws IOException;
+
+}

app/lib/jackson-mapper-lgpl-1.9.12.jar

@@ -0,0 +1,10 @@
+package processing.app.packages;
+
+import java.io.IOException;
+import java.io.Writer;
+
+public interface LibraryConverter {
+
+  void convert(Library library, Writer writer) throws IOException;
+
+}

app/src/processing/app/packages/DependenciesResolver.java

@@ -0,0 +1,85 @@
+package processing.app.packages.npm;
+
+import org.apache.commons.exec.CommandLine;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import processing.app.helpers.FileUtils;
+import processing.app.packages.DependenciesResolver;
+import processing.app.packages.Library;
+import processing.app.packages.LibraryConverter;
+import processing.app.tools.ExternalProcessExecutor;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileWriter;
+import java.io.IOException;
+
+public class NPMDependenciesResolver implements DependenciesResolver {
+
+  private final String pathToNpmExec;
+  private final LibraryConverter libraryConverter;
+  private final ObjectMapper objectMapper;
+
+  public NPMDependenciesResolver(String pathToNpmExec, LibraryConverter libraryConverter) {
+    this.pathToNpmExec = pathToNpmExec;
+    this.libraryConverter = libraryConverter;
+    this.objectMapper = new ObjectMapper();
+  }
+
+  @Override
+  public void resolveAndDownload(File libFolder, File targetFolder) throws IOException {
+    Library library = Library.create(libFolder);
+    File packageJson = new File(targetFolder, "package.json");
+
+    FileWriter fw = null;
+    try {
+      fw = new FileWriter(packageJson);
+      libraryConverter.convert(library, fw);
+    } finally {
+      if (fw != null) {
+        fw.close();
+      }
+    }
+
+    ExternalProcessExecutor executor = new ExternalProcessExecutor();
+    executor.setWorkingDirectory(targetFolder);
+    CommandLine parse = CommandLine.parse(pathToNpmExec + " install -d");
+    executor.execute(parse);
+
+    recursivelyMoveNodeModulesToUpperFolder(targetFolder, targetFolder);
+  }
+
+  private void recursivelyMoveNodeModulesToUpperFolder(File currentFolder, File targetFolder) throws IOException {
+    File subNodeModules = new File(currentFolder, "node_modules");
+    if (subNodeModules.exists() && subNodeModules.isDirectory()) {
+      recursivelyMoveNodeModulesToUpperFolder(subNodeModules, targetFolder);
+      FileUtils.recursiveDelete(subNodeModules);
+      return;
+    }
+
+    File[] nodeModules = currentFolder.listFiles(new FileFilter() {
+      @Override
+      public boolean accept(File file) {
+        return file.isDirectory() && !file.isHidden();
+      }
+    });
+
+    for (File nodeModule : nodeModules) {
+      subNodeModules = new File(nodeModule, "node_modules");
+      if (subNodeModules.exists() && subNodeModules.isDirectory()) {
+        recursivelyMoveNodeModulesToUpperFolder(subNodeModules, targetFolder);
+        FileUtils.recursiveDelete(subNodeModules);
+      }
+      JsonNode packageJson = objectMapper.readTree(new File(nodeModule, "package.json"));
+      String version = packageJson.get("version").getTextValue();
+      File dest = new File(targetFolder, nodeModule.getName() + "-" + version);
+      if (!dest.exists()) {
+        if (!nodeModule.renameTo(dest)) {
+          throw new IOException("Unable to move " + nodeModule + " to " + dest);
+        }
+      } else {
+        FileUtils.recursiveDelete(nodeModule);
+      }
+    }
+  }
+}

app/src/processing/app/packages/LibraryConverter.java

@@ -0,0 +1,39 @@
+package processing.app.packages.npm;
+
+import org.codehaus.jackson.map.ObjectMapper;
+import processing.app.packages.Library;
+import processing.app.packages.LibraryConverter;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+public class NPMLibraryConverter implements LibraryConverter {
+
+  @Override
+  public void convert(Library library, Writer writer) throws IOException {
+    Map<String, Object> packageJson = new HashMap<String, Object>();
+    packageJson.put("name", library.getName());
+    packageJson.put("version", "1.0.0");
+
+    Map<String, Object> dependencies = new HashMap<String, Object>();
+    for (String dependency : library.getDependencies()) {
+      String name;
+      String version = "*";
+      if (dependency.contains("(")) {
+        name = dependency.substring(0, dependency.indexOf("(")).trim().toLowerCase(Locale.US);
+        version = dependency.substring(dependency.indexOf("(") + 1, dependency.indexOf(")"));
+      } else {
+        name = dependency.trim().toLowerCase(Locale.US);
+      }
+      dependencies.put(name, version);
+    }
+
+    packageJson.put("dependencies", dependencies);
+
+    ObjectMapper objectMapper = new ObjectMapper();
+    objectMapper.writeValue(writer, packageJson);
+  }
+}

app/src/processing/app/packages/npm/NPMDependenciesResolver.java

@@ -12,22 +12,44 @@
  */
 public class ExternalProcessExecutor extends DefaultExecutor {
 
-  public ExternalProcessExecutor(final OutputStream os) {
+  public ExternalProcessExecutor() {
+    this(false);
+  }
+
+  public ExternalProcessExecutor(boolean debug) {
+    this(null, null, debug);
+  }
+
+  public ExternalProcessExecutor(OutputStream stdout) {
+    this(stdout, null, false);
+  }
+
+  public ExternalProcessExecutor(final OutputStream stdout, final OutputStream stderr, final boolean debug) {
     this.setStreamHandler(new ExecuteStreamHandler() {
       @Override
       public void setProcessInputStream(OutputStream outputStream) throws IOException {
       }
 
       @Override
       public void setProcessErrorStream(InputStream inputStream) throws IOException {
+        pipe(inputStream, stderr);
       }
 
       @Override
       public void setProcessOutputStream(InputStream inputStream) throws IOException {
+        pipe(inputStream, stdout);
+      }
+
+      private void pipe(InputStream inputStream, OutputStream output) throws IOException {
         byte[] buf = new byte[4096];
-        int bytes = -1;
+        int bytes;
         while ((bytes = inputStream.read(buf)) != -1) {
-          os.write(buf, 0, bytes);
+          if (debug) {
+            System.out.println(new String(buf, 0, bytes));
+          }
+          if (output != null) {
+            output.write(buf, 0, bytes);
+          }
         }
       }
 

app/src/processing/app/packages/npm/NPMLibraryConverter.java

@@ -0,0 +1,65 @@
+package processing.app.packages.npm;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import processing.app.helpers.FileUtils;
+import processing.app.packages.DependenciesResolver;
+
+import java.io.File;
+import java.io.FileFilter;
+
+import static org.junit.Assert.assertTrue;
+
+public class NPMDependenciesResolverTest {
+
+  private DependenciesResolver resolver;
+  private File targetFolder;
+
+  @Before
+  public void setUp() throws Exception {
+    File parent = new File(NPMDependenciesResolverTest.class.getResource("/").getFile());
+    String pathToNpmExec = new File(parent, "../../build/linux/work/nodejs/bin/npm").getAbsolutePath();
+
+    resolver = new NPMDependenciesResolver(pathToNpmExec, new NPMLibraryConverter());
+
+    targetFolder = new File(System.getProperty("java.io.tmpdir"), "arduino_" + Math.round(Math.random() * 100000));
+    targetFolder.mkdirs();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    FileUtils.recursiveDelete(targetFolder);
+  }
+
+  @Test
+  public void shouldDownloadAndOrganizeDependencies() throws Exception {
+    File libFolder = new File(NPMLibraryConverterTest.class.getResource("/processing/app/packages/test_lib_node_modules/library.properties").getFile()).getParentFile();
+
+    resolver.resolveAndDownload(libFolder, targetFolder);
+
+    assertTrue(listFoldersStartingWith("async").length >= 1);
+    assertTrue(listFoldersStartingWith("orientdb").length >= 1);
+    assertTrue(listFoldersStartingWith("by-mocha").length >= 1);
+    assertTrue(listFoldersStartingWith("grunt-mocha-cli").length >= 1);
+
+    File[] folders = targetFolder.listFiles(new FileFilter() {
+      @Override
+      public boolean accept(File file) {
+        return file.isDirectory();
+      }
+    });
+
+    // modules we depend to have sub dependencies
+    assertTrue(folders.length > 4);
+  }
+
+  private File[] listFoldersStartingWith(final String prefix) {
+    return targetFolder.listFiles(new FileFilter() {
+      @Override
+      public boolean accept(File file) {
+        return file.isDirectory() && file.getName().startsWith(prefix);
+      }
+    });
+  }
+}

app/src/processing/app/tools/ExternalProcessExecutor.java

@@ -0,0 +1,24 @@
+package processing.app.packages.npm;
+
+import org.junit.Test;
+import processing.app.packages.Library;
+
+import java.io.File;
+import java.io.StringWriter;
+
+import static org.junit.Assert.assertEquals;
+
+public class NPMLibraryConverterTest {
+
+  @Test
+  public void shouldConvertToPackageJSON() throws Exception {
+    File libFolder = new File(NPMLibraryConverterTest.class.getResource("/processing/app/packages/test_lib/library.properties").getFile()).getParentFile();
+    Library library = Library.create(libFolder);
+
+    NPMLibraryConverter converter = new NPMLibraryConverter();
+    StringWriter writer = new StringWriter();
+    converter.convert(library, writer);
+
+    assertEquals("{\"dependencies\":{\"ethernet\":\">=1.0\",\"by-mocha\":\"*\",\"spi\":\"*\",\"servo\":\"=3.0\",\"grunt-mocha-cli\":\"*\"},\"name\":\"WebServer\",\"version\":\"1.0.0\"}", writer.toString());
+  }
+}

app/test/processing/app/packages/npm/NPMDependenciesResolverTest.java

@@ -0,0 +1,10 @@
+name=WebServer
+author=cmaglie
+email=Cristian Maglie <[email protected]>
+sentence=A library that makes coding Webserver a breeze.
+paragraph=Supports HTTP1.1 and you can do GET and POST.
+url=http://asdasd.com/
+architectures=avr,sam
+version=1.0
+dependencies=Ethernet (>=1.0),Servo (=2.0),Servo (=3.0),SPI,,by-mocha,grunt-mocha-cli
+core-dependencies=arduino (>=1.5.0)