Merge pull request #11487 from cmaglie/do-not-delete-unused-packages-index

Do not delete unused 3rd party index files

Author: cmaglie

app/src/cc/arduino/contributions/packages/ui/ContributionManagerUI.java

@@ -140,9 +140,7 @@ public void onUpdatePressed() {
     installerThread = new Thread(() -> {
       try {
         setProgressVisible(true, "");
-        List<String> downloadedPackageIndexFiles = installer
-            .updateIndex(this::setProgress);
-        installer.deleteUnknownFiles(downloadedPackageIndexFiles);
+        installer.updateIndex(this::setProgress);
         onIndexesUpdated();
         if (contribTable.getCellEditor() != null) {
           contribTable.getCellEditor().stopCellEditing();

app/src/processing/app/Base.java

@@ -315,8 +315,7 @@ public Base(String[] args) throws Exception {
           BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier);
       ProgressListener progressListener = new ConsoleProgressListener();
 
-      List<String> downloadedPackageIndexFiles = contributionInstaller.updateIndex(progressListener);
-      contributionInstaller.deleteUnknownFiles(downloadedPackageIndexFiles);
+      contributionInstaller.updateIndex(progressListener);
       indexer.parseIndex();
       indexer.syncWithFilesystem();
 

arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java

@@ -41,7 +41,6 @@
 import org.apache.commons.exec.DefaultExecutor;
 import org.apache.commons.exec.Executor;
 import org.apache.commons.exec.PumpStreamHandler;
-import org.apache.commons.io.FilenameUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import processing.app.BaseNoGui;
@@ -284,7 +283,7 @@ public synchronized List<String> remove(ContributedPlatform contributedPlatform)
     return errors;
   }
 
-  public synchronized List<String> updateIndex(ProgressListener progressListener) {
+  public synchronized void updateIndex(ProgressListener progressListener) {
     MultiStepProgress progress = new MultiStepProgress(1);
 
     final DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.indexer.getStagingFolder());
@@ -293,14 +292,11 @@ public synchronized List<String> updateIndex(ProgressListener progressListener)
       PreferencesData.getCollection(Constants.PREF_BOARDS_MANAGER_ADDITIONAL_URLS)
     );
     packageIndexURLs.add(Constants.PACKAGE_INDEX_URL);
-    List<String> downloadedPackageIndexFilesAccumulator = new LinkedList<>();
 
     for (String packageIndexURLString : packageIndexURLs) {
       try {
         // Extract the file name from the URL
         final URL packageIndexURL = new URL(packageIndexURLString);
-        String indexFileName = FilenameUtils.getName(packageIndexURL.getPath());
-        downloadedPackageIndexFilesAccumulator.add(BaseNoGui.indexer.getIndexFile(indexFileName).getName());
 
         log.info("Start download and signature check of={}", packageIndexURLs);
         downloader.downloadIndexAndSignature(progress, packageIndexURL, progressListener, signatureVerifier);
@@ -312,22 +308,5 @@ public synchronized List<String> updateIndex(ProgressListener progressListener)
 
     progress.stepDone();
     log.info("Downloaded package index URL={}", packageIndexURLs);
-    return downloadedPackageIndexFilesAccumulator;
-  }
-
-  public synchronized void deleteUnknownFiles(List<String> downloadedPackageIndexFiles) throws IOException {
-    File preferencesFolder = BaseNoGui.indexer.getIndexFile(".").getParentFile();
-    File[] additionalPackageIndexFiles = preferencesFolder.listFiles(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME));
-    if (additionalPackageIndexFiles == null) {
-      return;
-    }
-    log.info("Check unknown files. Additional package index folder files={}, Additional package index url downloaded={}", downloadedPackageIndexFiles, additionalPackageIndexFiles);
-
-    for (File additionalPackageIndexFile : additionalPackageIndexFiles) {
-      if (!downloadedPackageIndexFiles.contains(additionalPackageIndexFile.getName())) {
-        log.info("Delete this unknown file={} because not included in this list={}", additionalPackageIndexFile, additionalPackageIndexFiles);
-        Files.delete(additionalPackageIndexFile.toPath());
-      }
-    }
   }
 }

arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java

@@ -40,7 +40,7 @@
 
 public class ContributionsIndex {
 
-  private ArrayList<ContributedPackage> packages = new ArrayList<ContributedPackage>();
+  private ArrayList<ContributedPackage> packages = new ArrayList<>();
   public List<ContributedPackage> getPackages() { return packages; }
 
   public ContributedPackage findPackage(String packageName) {

arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java

@@ -36,6 +36,7 @@
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.FilenameUtils;
 
 import processing.app.BaseNoGui;
 import processing.app.Platform;
@@ -50,6 +51,8 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -107,19 +110,14 @@ public void parseIndex() throws Exception {
     index.getPackages().forEach(pack -> pack.setTrusted(true));
 
     // Overlay 3rd party indexes
-    File[] indexFiles = preferencesFolder.listFiles(new TestPackageIndexFilenameFilter(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME)));
-
-    if (indexFiles != null) {
-      for (File indexFile : indexFiles) {
-        try {
-          mergeContributions(indexFile);
-        } catch (JsonProcessingException e) {
-          System.err.println(format(tr("Skipping contributed index file {0}, parsing error occured:"), indexFile));
-          System.err.println(e);
-        }
+    List<File> indexFiles = get3rdPartyIndexFiles();
+    for (File indexFile : indexFiles) {
+      try {
+        mergeContributions(indexFile);
+      } catch (JsonProcessingException e) {
+        System.err.println(format(tr("Skipping contributed index file {0}, parsing error occured:"), indexFile));
+        System.err.println(e);
       }
-    } else {
-      System.err.println(format(tr("Error reading package indexes folder: {0}\n(maybe a permission problem?)"), preferencesFolder));
     }
 
     // Fill tools and toolsDependency cross references
@@ -146,6 +144,29 @@ public void parseIndex() throws Exception {
     index.fillCategories();
   }
 
+  private List<File> get3rdPartyIndexFiles() throws MalformedURLException {
+    List<File> indexFiles = new ArrayList<>();
+    for (String urlString : PreferencesData.getCollection(Constants.PREF_BOARDS_MANAGER_ADDITIONAL_URLS)) {
+      final URL url = new URL(urlString);
+      String filename = FilenameUtils.getName(url.getPath());
+      indexFiles.add(getIndexFile(filename));
+    }
+
+    File[] testIndexFiles = preferencesFolder.listFiles((dir, name) -> {
+      if (!new File(dir, name).isFile())
+        return false;
+      if (!name.startsWith("test_package_") || !name.endsWith("_index.json"))
+        return false;
+      return true;
+    });
+    if (testIndexFiles == null) {
+      System.err.println(
+          format(tr("Error reading package indexes folder: {0}\n(maybe a permission problem?)"), preferencesFolder));
+    }
+    indexFiles.addAll(Arrays.asList(testIndexFiles));
+    return indexFiles;
+  }
+
   private void mergeContributions(File indexFile) throws IOException {
     if (!indexFile.exists())
       return;