From 9669d67e166055459bd446c21cb92b157e695e52 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 5 Aug 2016 17:05:44 +0200 Subject: [PATCH 1/7] Revert to avrdude 6.0.1 This reverts commits: 385edd26fc2b64a442130f2f6a63e22416621857 (Removed "macosx-fix-bundled-toolchain-missing-symlinks" build task) f5fd6423c3df9423bf285306b761f3a54d918dcf (Updated avrdude to 6.3-arduino2) 5a01929b8fdc10dfca974d83bb8065696384f1ec (avrdude 6.3: fixed IDE build for macosx) And part of this commit: 05b647721cec48f1b6663688b61f4408539795a1 (Update avr-gcc to 4.9.2 and avrdude to 6.3) --- build/build.xml | 35 ++++++++++++------- ...5-armhf-pc-linux-gnu-glibc2.13.tar.bz2.sha | 1 + ...0.1-arduino5-i686-pc-linux-gnu.tar.bz2.sha | 1 + ...1-arduino5-x86_64-pc-linux-gnu.tar.bz2.sha | 1 + ....0-arduino2-armhf-pc-linux-gnu.tar.bz2.sha | 1 - ...3.0-arduino2-i686-pc-linux-gnu.tar.bz2.sha | 1 - ...0-arduino2-x86_64-pc-linux-gnu.tar.bz2.sha | 1 - ...rduino5-r2-i386-apple-darwin11.tar.bz2.sha | 1 + ...0-arduino2-i386-apple-darwin11.tar.bz2.sha | 1 - build/shared/revisions.txt | 2 ++ ...vrdude-6.0.1-arduino5-i686-mingw32.zip.sha | 1 + 11 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 build/linux/avrdude-6.0.1-arduino5-armhf-pc-linux-gnu-glibc2.13.tar.bz2.sha create mode 100644 build/linux/avrdude-6.0.1-arduino5-i686-pc-linux-gnu.tar.bz2.sha create mode 100644 build/linux/avrdude-6.0.1-arduino5-x86_64-pc-linux-gnu.tar.bz2.sha delete mode 100644 build/linux/avrdude-6.3.0-arduino2-armhf-pc-linux-gnu.tar.bz2.sha delete mode 100644 build/linux/avrdude-6.3.0-arduino2-i686-pc-linux-gnu.tar.bz2.sha delete mode 100644 build/linux/avrdude-6.3.0-arduino2-x86_64-pc-linux-gnu.tar.bz2.sha create mode 100644 build/macosx/avrdude-6.0.1-arduino5-r2-i386-apple-darwin11.tar.bz2.sha delete mode 100644 build/macosx/avrdude-6.3.0-arduino2-i386-apple-darwin11.tar.bz2.sha create mode 100644 build/windows/avrdude-6.0.1-arduino5-i686-mingw32.zip.sha diff --git a/build/build.xml b/build/build.xml index 4723bcda7ad..4fa7add32fb 100644 --- a/build/build.xml +++ b/build/build.xml @@ -92,7 +92,6 @@ - @@ -494,8 +493,8 @@ - - + + @@ -719,8 +718,8 @@ - - + + @@ -751,8 +750,8 @@ - - + + @@ -783,8 +782,8 @@ - - + + @@ -1071,8 +1070,8 @@ - - + + @@ -1161,7 +1160,8 @@ - + + @@ -1191,6 +1191,17 @@ + + + + + + + + + + + ${BUILD_DATE} diff --git a/build/linux/avrdude-6.0.1-arduino5-armhf-pc-linux-gnu-glibc2.13.tar.bz2.sha b/build/linux/avrdude-6.0.1-arduino5-armhf-pc-linux-gnu-glibc2.13.tar.bz2.sha new file mode 100644 index 00000000000..6d87ebcafe9 --- /dev/null +++ b/build/linux/avrdude-6.0.1-arduino5-armhf-pc-linux-gnu-glibc2.13.tar.bz2.sha @@ -0,0 +1 @@ +b5968416647e79f5c395bf2a9ba000127dacc9da diff --git a/build/linux/avrdude-6.0.1-arduino5-i686-pc-linux-gnu.tar.bz2.sha b/build/linux/avrdude-6.0.1-arduino5-i686-pc-linux-gnu.tar.bz2.sha new file mode 100644 index 00000000000..0bc23f68811 --- /dev/null +++ b/build/linux/avrdude-6.0.1-arduino5-i686-pc-linux-gnu.tar.bz2.sha @@ -0,0 +1 @@ +cf859c679713cae2f1b9089d797d211dd84fe658 diff --git a/build/linux/avrdude-6.0.1-arduino5-x86_64-pc-linux-gnu.tar.bz2.sha b/build/linux/avrdude-6.0.1-arduino5-x86_64-pc-linux-gnu.tar.bz2.sha new file mode 100644 index 00000000000..e4b0302e0fa --- /dev/null +++ b/build/linux/avrdude-6.0.1-arduino5-x86_64-pc-linux-gnu.tar.bz2.sha @@ -0,0 +1 @@ +cfc52036f532c3a7050f6d4ec7645d060800aa39 diff --git a/build/linux/avrdude-6.3.0-arduino2-armhf-pc-linux-gnu.tar.bz2.sha b/build/linux/avrdude-6.3.0-arduino2-armhf-pc-linux-gnu.tar.bz2.sha deleted file mode 100644 index 389ab1832ad..00000000000 --- a/build/linux/avrdude-6.3.0-arduino2-armhf-pc-linux-gnu.tar.bz2.sha +++ /dev/null @@ -1 +0,0 @@ -a074c17e2013942e3e081fe03875c9d5701a0922 diff --git a/build/linux/avrdude-6.3.0-arduino2-i686-pc-linux-gnu.tar.bz2.sha b/build/linux/avrdude-6.3.0-arduino2-i686-pc-linux-gnu.tar.bz2.sha deleted file mode 100644 index 844fb6a27cc..00000000000 --- a/build/linux/avrdude-6.3.0-arduino2-i686-pc-linux-gnu.tar.bz2.sha +++ /dev/null @@ -1 +0,0 @@ -e9ce49f5e5c7d0d2eabd9405ca6160b70cd52645 diff --git a/build/linux/avrdude-6.3.0-arduino2-x86_64-pc-linux-gnu.tar.bz2.sha b/build/linux/avrdude-6.3.0-arduino2-x86_64-pc-linux-gnu.tar.bz2.sha deleted file mode 100644 index 1104a38421e..00000000000 --- a/build/linux/avrdude-6.3.0-arduino2-x86_64-pc-linux-gnu.tar.bz2.sha +++ /dev/null @@ -1 +0,0 @@ -3ba4c2660f8a8c40ba3902ab8a158aa3e1b91764 diff --git a/build/macosx/avrdude-6.0.1-arduino5-r2-i386-apple-darwin11.tar.bz2.sha b/build/macosx/avrdude-6.0.1-arduino5-r2-i386-apple-darwin11.tar.bz2.sha new file mode 100644 index 00000000000..3b1625eee60 --- /dev/null +++ b/build/macosx/avrdude-6.0.1-arduino5-r2-i386-apple-darwin11.tar.bz2.sha @@ -0,0 +1 @@ +52afce6cbe1cc1021bca1234cf25b298350f97dd diff --git a/build/macosx/avrdude-6.3.0-arduino2-i386-apple-darwin11.tar.bz2.sha b/build/macosx/avrdude-6.3.0-arduino2-i386-apple-darwin11.tar.bz2.sha deleted file mode 100644 index edb160cf3ea..00000000000 --- a/build/macosx/avrdude-6.3.0-arduino2-i386-apple-darwin11.tar.bz2.sha +++ /dev/null @@ -1 +0,0 @@ -39a1940fbb4a25af5316d9a6bde99b6a29585e37 diff --git a/build/shared/revisions.txt b/build/shared/revisions.txt index 304f7ac41b3..96b272f85cf 100644 --- a/build/shared/revisions.txt +++ b/build/shared/revisions.txt @@ -4,6 +4,8 @@ ARDUINO 1.6.11 * Fixed a serious bug that prevented some 3rd party boards, installed through external index.json URL, to work correctly. * builder: fixed regression about inclusion of files in subfolders of the sketch (see https://github.com/arduino/Arduino/issues/5186 for details) +* avrdude: reverted to version 6.0.1, until all discovered regressions are solved + (see https://github.com/arduino/Arduino/issues?q=is%3Aissue+is%3Aopen+label%3A%22Component%3A+Avrdude+6.3%22 for details) ARDUINO 1.6.10 - 2016.07.26 diff --git a/build/windows/avrdude-6.0.1-arduino5-i686-mingw32.zip.sha b/build/windows/avrdude-6.0.1-arduino5-i686-mingw32.zip.sha new file mode 100644 index 00000000000..e55aa9d558b --- /dev/null +++ b/build/windows/avrdude-6.0.1-arduino5-i686-mingw32.zip.sha @@ -0,0 +1 @@ +bad246afeb0b9bcbe623a29cb204127f901102cf From 7008f6c57c11a2f6da585d9961b72ff27b461789 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 11 Aug 2016 10:41:48 +0200 Subject: [PATCH 2/7] ContributionsIndexer now has bundled hardware path as a field --- app/src/processing/app/Base.java | 6 ++++-- .../packages/ContributionsIndexer.java | 18 ++++++++++-------- arduino-core/src/processing/app/BaseNoGui.java | 5 +++-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/app/src/processing/app/Base.java b/app/src/processing/app/Base.java index d45d9c638f2..95becc3b0b2 100644 --- a/app/src/processing/app/Base.java +++ b/app/src/processing/app/Base.java @@ -344,13 +344,15 @@ public Base(String[] args) throws Exception { PreferencesData.save(); if (parser.isInstallBoard()) { - ContributionsIndexer indexer = new ContributionsIndexer(BaseNoGui.getSettingsFolder(), BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier()); + ContributionsIndexer indexer = new ContributionsIndexer( + BaseNoGui.getSettingsFolder(), BaseNoGui.getHardwareFolder(), + BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier()); ProgressListener progressListener = new ConsoleProgressListener(); List downloadedPackageIndexFiles = contributionInstaller.updateIndex(progressListener); contributionInstaller.deleteUnknownFiles(downloadedPackageIndexFiles); indexer.parseIndex(); - indexer.syncWithFilesystem(BaseNoGui.getHardwareFolder()); + indexer.syncWithFilesystem(); String[] boardToInstallParts = parser.getBoardToInstall().split(":"); diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java index 5e8fb03ab95..917d940148a 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java @@ -66,12 +66,14 @@ public class ContributionsIndexer { private final File packagesFolder; private final File stagingFolder; private final File preferencesFolder; + private final File builtInHardwareFolder; private final Platform platform; private final SignatureVerifier signatureVerifier; private ContributionsIndex index; - public ContributionsIndexer(File preferencesFolder, Platform platform, SignatureVerifier signatureVerifier) { + public ContributionsIndexer(File preferencesFolder, File builtInHardwareFolder, Platform platform, SignatureVerifier signatureVerifier) { this.preferencesFolder = preferencesFolder; + this.builtInHardwareFolder = builtInHardwareFolder; this.platform = platform; this.signatureVerifier = signatureVerifier; packagesFolder = new File(preferencesFolder, "packages"); @@ -188,22 +190,22 @@ private ContributionsIndex parseIndex(File indexFile) throws IOException { } } - public void syncWithFilesystem(File hardwareFolder) throws IOException { - syncBuiltInHardwareFolder(hardwareFolder); + public void syncWithFilesystem() throws IOException { + syncBuiltInHardware(); - syncLocalPackagesFolder(); + syncLocalPackages(); } - private void syncBuiltInHardwareFolder(File hardwareFolder) throws IOException { + private void syncBuiltInHardware() throws IOException { if (index == null) { return; } - for (File folder : hardwareFolder.listFiles(ONLY_DIRS)) { + for (File folder : builtInHardwareFolder.listFiles(ONLY_DIRS)) { ContributedPackage pack = index.findPackage(folder.getName()); if (pack != null) { syncBuiltInPackageWithFilesystem(pack, folder); - File toolsFolder = new File(hardwareFolder, "tools"); + File toolsFolder = new File(builtInHardwareFolder, "tools"); if (toolsFolder.isDirectory()) { for (File toolFolder : toolsFolder.listFiles(ONLY_DIRS)) { File builtInToolsMetadata = new File(toolFolder, "builtin_tools_versions.txt"); @@ -231,7 +233,7 @@ private void syncBuiltInPackageWithFilesystem(ContributedPackage pack, File hard } } - private void syncLocalPackagesFolder() { + private void syncLocalPackages() { if (!packagesFolder.isDirectory()) { return; } diff --git a/arduino-core/src/processing/app/BaseNoGui.java b/arduino-core/src/processing/app/BaseNoGui.java index 43b99aff8b8..1cef43cbf83 100644 --- a/arduino-core/src/processing/app/BaseNoGui.java +++ b/arduino-core/src/processing/app/BaseNoGui.java @@ -608,7 +608,8 @@ static public void initLogger() { } static public void initPackages() throws Exception { - indexer = new ContributionsIndexer(BaseNoGui.getSettingsFolder(), BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier()); + indexer = new ContributionsIndexer(getSettingsFolder(), getHardwareFolder(), getPlatform(), + new GPGDetachedSignatureVerifier()); File indexFile = indexer.getIndexFile("package_index.json"); File defaultPackageJsonFile = new File(getContentFile("dist"), "package_index.json"); if (!indexFile.isFile() || (defaultPackageJsonFile.isFile() && defaultPackageJsonFile.lastModified() > indexFile.lastModified())) { @@ -637,7 +638,7 @@ static public void initPackages() throws Exception { FileUtils.deleteIfExists(indexSignatureFile); throw e; } - indexer.syncWithFilesystem(getHardwareFolder()); + indexer.syncWithFilesystem(); packages = new LinkedHashMap(); loadHardware(getHardwareFolder()); From e80c0859963f5f9d695743f4b7dc2d77ec18d4f7 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 11 Aug 2016 10:46:28 +0200 Subject: [PATCH 3/7] Use a specific hardware/package_index_bundled.json. AVR core version to 1.6.13 Previously, during the build, the full package_index.json was downloaded and distributed with the Arduino IDE. This lead to a situation where it was difficult to test new AVR cores before publishing them to the public package_index.json. Now the bundled AVR core is specificed in the file: `hardware/package_index_bundled.json` this index is loaded from the IDE at startup and the package_index.json is overlayed on it. This should also solve part of #5143 (Repeatable builds and snapshots of package/library indexes) --- arduino-core/src/cc/arduino/Constants.java | 1 + .../packages/ContributionsIndex.java | 4 - .../packages/ContributionsIndexer.java | 14 +- build/build.xml | 24 +-- hardware/arduino/avr/platform.txt | 2 +- hardware/package_index_bundled.json | 146 ++++++++++++++++++ 6 files changed, 167 insertions(+), 24 deletions(-) create mode 100644 hardware/package_index_bundled.json diff --git a/arduino-core/src/cc/arduino/Constants.java b/arduino-core/src/cc/arduino/Constants.java index 3c90abe7da6..80349d82dae 100644 --- a/arduino-core/src/cc/arduino/Constants.java +++ b/arduino-core/src/cc/arduino/Constants.java @@ -39,6 +39,7 @@ public class Constants { public static final String PREF_CONTRIBUTIONS_TRUST_ALL = "contributions.trust.all"; public static final String DEFAULT_INDEX_FILE_NAME = "package_index.json"; + public static final String BUNDLED_INDEX_FILE_NAME = "package_index_bundled.json"; public static final List PROTECTED_PACKAGE_NAMES = Arrays.asList("arduino", "Intel"); public static final String LIBRARY_DEVELOPMENT_FLAG_FILE = ".development"; diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java index 6f9295f74d6..1ffc97d0229 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java @@ -141,8 +141,4 @@ public String toString() { res += pack + "\n"; return res; } - - public void setTrusted() { - getPackages().stream().forEach(pack -> pack.setTrusted(true)); - } } diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java index 917d940148a..443987c0d17 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java @@ -82,12 +82,21 @@ public ContributionsIndexer(File preferencesFolder, File builtInHardwareFolder, public void parseIndex() throws Exception { File defaultIndexFile = getIndexFile(Constants.DEFAULT_INDEX_FILE_NAME); + File bundledIndexFile = new File(builtInHardwareFolder, Constants.BUNDLED_INDEX_FILE_NAME); + + // Check main index signature if (!PreferencesData.getBoolean("allow_insecure_packages") && !signatureVerifier.isSigned(defaultIndexFile)) { throw new SignatureVerificationFailedException(Constants.DEFAULT_INDEX_FILE_NAME); } - index = parseIndex(defaultIndexFile); - index.setTrusted(); + // Read bundled index and overlay the default index + index = parseIndex(bundledIndexFile); + mergeContributions(parseIndex(defaultIndexFile), defaultIndexFile); + + // Set main and bundled indexes as trusted + index.getPackages().forEach(pack -> pack.setTrusted(true)); + + // Overlay 3rd party indexes File[] indexFiles = preferencesFolder.listFiles(new TestPackageIndexFilenameFilter(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME))); for (File indexFile : indexFiles) { @@ -100,6 +109,7 @@ public void parseIndex() throws Exception { } } + // Fill tools and toolsDependency cross references List packages = index.getPackages(); Collection packagesWithTools = packages.stream() .filter(input -> input.getTools() != null && !input.getTools().isEmpty()) diff --git a/build/build.xml b/build/build.xml index 4fa7add32fb..83c09fda806 100644 --- a/build/build.xml +++ b/build/build.xml @@ -10,7 +10,6 @@ - @@ -274,7 +273,7 @@ - + @@ -286,7 +285,7 @@ - + @@ -295,8 +294,8 @@ - Mismatching versions for bundled AVR core and package_index.json. - Please check your platform.txt and package_index.json. + Mismatching versions for bundled AVR core and package_index_bundled.json. + Please check your platform.txt and package_index_bundled.json. @@ -1168,19 +1167,10 @@ - - - - - - - + + + diff --git a/hardware/arduino/avr/platform.txt b/hardware/arduino/avr/platform.txt index 83cf12b2f84..0738d798113 100644 --- a/hardware/arduino/avr/platform.txt +++ b/hardware/arduino/avr/platform.txt @@ -6,7 +6,7 @@ # https://github.com/arduino/Arduino/wiki/Arduino-IDE-1.5-3rd-party-Hardware-specification name=Arduino AVR Boards -version=1.6.12 +version=1.6.13 # AVR compile variables # --------------------- diff --git a/hardware/package_index_bundled.json b/hardware/package_index_bundled.json new file mode 100644 index 00000000000..4e301a285aa --- /dev/null +++ b/hardware/package_index_bundled.json @@ -0,0 +1,146 @@ +{ + "packages": [ + { + "name": "arduino", + "maintainer": "Arduino", + "websiteURL": "http://www.arduino.cc/", + "email": "packages@arduino.cc", + "help": { + "online": "http://www.arduino.cc/en/Reference/HomePage" + }, + "platforms": [ + { + "name": "Arduino AVR Boards", + "architecture": "avr", + "version": "1.6.13", + "category": "Arduino", + "help": { + "online": "http://www.arduino.cc/en/Reference/HomePage" + }, + "url": "http://downloads.arduino.cc/cores/avr-1.6.13.tar.bz2", + "archiveFileName": "avr-1.6.13.tar.bz2", + "checksum": "SHA-256:617f458dd3507072b9a6f9fdc78888c66aa420a2fb081c4c1556598a2d69d643", + "size": "4993644", + "boards": [ + {"name": "Arduino Yún"}, + {"name": "Arduino/Genuino Uno"}, + {"name": "Arduino Diecimila"}, + {"name": "Arduino Nano"}, + {"name": "Arduino/Genuino Mega"}, + {"name": "Arduino MegaADK"}, + {"name": "Arduino Leonardo"}, + {"name": "Arduino/Genuino Micro"}, + {"name": "Arduino Esplora"}, + {"name": "Arduino Mini"}, + {"name": "Arduino Ethernet"}, + {"name": "Arduino Fio"}, + {"name": "Arduino BT"}, + {"name": "Arduino LilyPadUSB"}, + {"name": "Arduino Lilypad"}, + {"name": "Arduino Pro"}, + {"name": "Arduino ATMegaNG"}, + {"name": "Arduino Robot Control"}, + {"name": "Arduino Robot Motor"}, + {"name": "Arduino Gemma"} + ], + "toolsDependencies": [ + { + "packager": "arduino", + "name": "avr-gcc", + "version": "4.9.2-atmel3.5.3-arduino2" + }, + { + "packager": "arduino", + "name": "avrdude", + "version": "6.0.1-arduino5" + } + ] + } + ], + "tools": [ + { + "name": "avrdude", + "version": "6.0.1-arduino5", + "systems": [ + { + "size": "267095", + "checksum": "SHA-256:23ea1341dbc117ec067f2eb1a498ad2bdd7d11fff0143c00b2e018c39804f6b4", + "host": "arm-linux-gnueabihf", + "archiveFileName": "avrdude-6.0.1-arduino5-armhf-pc-linux-gnu.tar.bz2", + "url": "http://downloads.arduino.cc/tools/avrdude-6.0.1-arduino5-armhf-pc-linux-gnu.tar.bz2" + }, + { + "size": "264894", + "checksum": "SHA-256:41af8d3b0a586853c8317b4fb5163ca0db594a1870ddf680fd988c42166fc3e5", + "host": "i386-apple-darwin11", + "archiveFileName": "avrdude-6.0.1-arduino5-i386-apple-darwin11.tar.bz2", + "url": "http://downloads.arduino.cc/tools/avrdude-6.0.1-arduino5-i386-apple-darwin11.tar.bz2" + }, + { + "size": "292629", + "checksum": "SHA-256:d826cca7383461f7e8adde686372cf900e9cb3afd639555cf2d6c645b283a476", + "host": "x86_64-linux-gnu", + "archiveFileName": "avrdude-6.0.1-arduino5-x86_64-pc-linux-gnu.tar.bz2", + "url": "http://downloads.arduino.cc/tools/avrdude-6.0.1-arduino5-x86_64-pc-linux-gnu.tar.bz2" + }, + { + "size": "283121", + "checksum": "SHA-256:5933d66927bce46ababa9b68a8b7f1d53f68c4f3ff7a5ce4b85d7cf4e6c6bfee", + "host": "i686-linux-gnu", + "archiveFileName": "avrdude-6.0.1-arduino5-i686-pc-linux-gnu.tar.bz2", + "url": "http://downloads.arduino.cc/tools/avrdude-6.0.1-arduino5-i686-pc-linux-gnu.tar.bz2" + }, + { + "size": "241634", + "checksum": "SHA-256:41f667f1f6a0ab8df46b4ffacd023176dcdef331d6db3b74bddd37d18cca0a44", + "host": "i686-mingw32", + "archiveFileName": "avrdude-6.0.1-arduino5-i686-mingw32.zip", + "url": "http://downloads.arduino.cc/tools/avrdude-6.0.1-arduino5-i686-mingw32.zip" + } + ] + }, + { + "name": "avr-gcc", + "version": "4.9.2-atmel3.5.3-arduino2", + "systems": [ + { + "size": "27400889", + "checksum": "SHA-256:77f300d519bc6b9a25df17b36cb303218e9a258c059b2f6bff8f71a0d8f96821", + "host": "arm-linux-gnueabihf", + "archiveFileName": "avr-gcc-4.9.2-atmel3.5.3-arduino2-armhf-pc-linux-gnu.tar.bz2", + "url": "http://downloads.arduino.cc/tools/avr-gcc-4.9.2-atmel3.5.3-arduino2-armhf-pc-linux-gnu.tar.bz2" + }, + { + "size": "27048070", + "checksum": "SHA-256:311258af188defe24a4b341e4e1f4dc93ca6c80516d3e3b55a2fc07a7050248b", + "host": "i386-apple-darwin11", + "archiveFileName": "avr-gcc-4.9.2-atmel3.5.3-arduino2-i386-apple-darwin11.tar.bz2", + "url": "http://downloads.arduino.cc/tools/avr-gcc-4.9.2-atmel3.5.3-arduino2-i386-apple-darwin11.tar.bz2" + }, + { + "size": "43847945", + "checksum": "SHA-256:f8e6ede8746c70be01ec79a30803277cd94360cc5b2e104762da0fbcf536fcc6", + "host": "i686-mingw32", + "archiveFileName": "avr-gcc-4.9.2-atmel3.5.3-arduino2-i686-mingw32.zip", + "url": "http://downloads.arduino.cc/tools/avr-gcc-4.9.2-atmel3.5.3-arduino2-i686-mingw32.zip" + }, + { + "size": "29292729", + "checksum": "SHA-256:f108951e7c4dc90926d1fc76cc27549f6ea63c702a2bb7ff39647a19ae86ec68", + "host": "i686-linux-gnu", + "archiveFileName": "avr-gcc-4.9.2-atmel3.5.3-arduino2-i686-pc-linux-gnu.tar.bz2", + "url": "http://downloads.arduino.cc/tools/avr-gcc-4.9.2-atmel3.5.3-arduino2-i686-pc-linux-gnu.tar.bz2" + }, + { + "size": "29882960", + "checksum": "SHA-256:3903a6d1bb9fdd91727e504b5993d5501f119bcb7f99f7aee98a2101e5629188", + "host": "x86_64-linux-gnu", + "archiveFileName": "avr-gcc-4.9.2-atmel3.5.3-arduino2-x86_64-pc-linux-gnu.tar.bz2", + "url": "http://downloads.arduino.cc/tools/avr-gcc-4.9.2-atmel3.5.3-arduino2-x86_64-pc-linux-gnu.tar.bz2" + } + ] + } + ] + } + ] +} From b1f9164c4c5e6b80247689e6094945be6a344404 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 11 Aug 2016 13:33:08 +0200 Subject: [PATCH 4/7] Slightly refactored ContributionsIndexer.syncBuiltInHardware() This is just a small rewrite of the function in a more clear way, no change in behavior. --- .../packages/ContributionsIndexer.java | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java index 443987c0d17..13ed66a2646 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java @@ -212,20 +212,28 @@ private void syncBuiltInHardware() throws IOException { } for (File folder : builtInHardwareFolder.listFiles(ONLY_DIRS)) { ContributedPackage pack = index.findPackage(folder.getName()); - if (pack != null) { - syncBuiltInPackageWithFilesystem(pack, folder); - - File toolsFolder = new File(builtInHardwareFolder, "tools"); - if (toolsFolder.isDirectory()) { - for (File toolFolder : toolsFolder.listFiles(ONLY_DIRS)) { - File builtInToolsMetadata = new File(toolFolder, "builtin_tools_versions.txt"); - if (builtInToolsMetadata.isFile()) { - PreferencesMap toolsMetadata = new PreferencesMap(builtInToolsMetadata).subTree(pack.getName()); - for (Map.Entry toolMetadata : toolsMetadata.entrySet()) { - syncToolWithFilesystem(pack, toolFolder, toolMetadata.getKey(), toolMetadata.getValue()); - } - } - } + if (pack == null) + continue; + syncBuiltInPackageWithFilesystem(pack, folder); + + File toolsFolder = new File(builtInHardwareFolder, "tools"); + if (!toolsFolder.isDirectory()) + continue; + + for (File toolFolder : toolsFolder.listFiles(ONLY_DIRS)) { + + // builtin_tools_versions.txt contains tools versions in the format: + // "PACKAGER.TOOL_NAME=TOOL_VERSION" + // for example: + // "arduino.avrdude=6.0.1-arduino5" + + File versionsFile = new File(toolFolder, "builtin_tools_versions.txt"); + if (!versionsFile.isFile()) + continue; + PreferencesMap toolsVersion = new PreferencesMap(versionsFile).subTree(pack.getName()); + for (String name : toolsVersion.keySet()) { + String version = toolsVersion.get(name); + syncToolWithFilesystem(pack, toolFolder, name, version); } } } From d8470e59f4f3cd66a7d050d28cd154ace92fb687 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 11 Aug 2016 13:39:56 +0200 Subject: [PATCH 5/7] Mark built-in tools as readonly and do not remove them when uninstalling This covers a very convoluted use-case that may be reproduce this way: 1. Using an previous version of the IDE, a new AVR core is installed using the board manager. 2. The IDE is then updated so the core installed in 1. is now also the bundled one 3. The AVR core installed 1. is now removed using the board manager 4. The board manager will uninstall the (presumably) no longer used tools, from the built-in folder leaving, in fact, the IDE without the bundled tools that are supposed to be read-only. This commit fix this bug by actually making the built-in tool read-only --- .../packages/ContributionInstaller.java | 9 +++++++-- .../contributions/packages/ContributionsIndexer.java | 12 ++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java index 47f895be638..28c3e0722d5 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java @@ -249,11 +249,16 @@ public synchronized List remove(ContributedPlatform contributedPlatform) // Check if the tools are no longer needed for (ContributedTool tool : contributedPlatform.getResolvedTools()) { - if (BaseNoGui.indexer.isContributedToolUsed(contributedPlatform, tool)) { + // Do not remove used tools + if (BaseNoGui.indexer.isContributedToolUsed(contributedPlatform, tool)) continue; - } + // Do not remove built-in tools DownloadableContribution toolContrib = tool.getDownloadableContribution(platform); + if (toolContrib.isReadOnly()) + continue; + + // Ok, delete the tool File destFolder = toolContrib.getInstalledFolder(); FileUtils.recursiveDelete(destFolder); toolContrib.setInstalled(false); diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java index 13ed66a2646..0e6d03794d8 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java @@ -233,7 +233,9 @@ private void syncBuiltInHardware() throws IOException { PreferencesMap toolsVersion = new PreferencesMap(versionsFile).subTree(pack.getName()); for (String name : toolsVersion.keySet()) { String version = toolsVersion.get(name); - syncToolWithFilesystem(pack, toolFolder, name, version); + DownloadableContribution tool = syncToolWithFilesystem(pack, toolFolder, name, version); + if (tool != null) + tool.setReadOnly(true); } } } @@ -292,21 +294,23 @@ private void syncPackageWithFilesystem(ContributedPackage pack, File root) { } } - private void syncToolWithFilesystem(ContributedPackage pack, File installationFolder, String toolName, String version) { + private DownloadableContribution syncToolWithFilesystem(ContributedPackage pack, File installationFolder, String toolName, String version) { ContributedTool tool = pack.findTool(toolName, version); if (tool == null) { tool = pack.findResolvedTool(toolName, version); } if (tool == null) { - return; + return null; } DownloadableContribution contrib = tool.getDownloadableContribution(platform); if (contrib == null) { System.err.println(tool + " seems to have no downloadable contributions for your operating system, but it is installed in\n" + installationFolder); - return; + return null; } contrib.setInstalled(true); contrib.setInstalledFolder(installationFolder); + contrib.setReadOnly(false); + return contrib; } private ContributedPlatform syncHardwareWithFilesystem(ContributedPackage pack, File installationFolder, String architecture, String version) { From e731fe026f87982473137f5baffd1387eb1003e7 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 11 Aug 2016 15:11:30 +0200 Subject: [PATCH 6/7] Boards Manager now install tools even if they are available in the IDE bundle Previously if a 3rd party core would require a tool already bundled in the IDE then boards manager skipped the installation of that tool. This is could lead to missing tools if the IDE is upgraded and the bundled tools may change. This patch fixes the bug by always installing tools when needed, even if they are already bundled. --- .../packages/ContributionInstaller.java | 15 +++++++-------- .../packages/ContributionsIndexer.java | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java index 28c3e0722d5..bbf9fab9fe9 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java @@ -79,16 +79,15 @@ public synchronized List install(ContributedPlatform contributedPlatform } // Do not download already installed tools - List tools = new LinkedList<>(contributedPlatform.getResolvedTools()); - Iterator toolsIterator = tools.iterator(); - while (toolsIterator.hasNext()) { - ContributedTool tool = toolsIterator.next(); + List tools = new ArrayList<>(); + for (ContributedTool tool : contributedPlatform.getResolvedTools()) { DownloadableContribution downloadable = tool.getDownloadableContribution(platform); if (downloadable == null) { throw new Exception(format(tr("Tool {0} is not available for your operating system."), tool.getName())); } - if (downloadable.isInstalled()) { - toolsIterator.remove(); + // Download the tool if it's not installed or it's a built-in tool + if (!downloadable.isInstalled() || downloadable.isReadOnly()) { + tools.add(tool); } } @@ -125,10 +124,10 @@ public synchronized List install(ContributedPlatform contributedPlatform List> resolvedToolReferences = contributedPlatform.getResolvedToolReferences().entrySet() .stream() - .filter((entry) -> !entry.getValue().getDownloadableContribution(platform).isInstalled()) + .filter((entry) -> !entry.getValue().getDownloadableContribution(platform).isInstalled() + || entry.getValue().getDownloadableContribution(platform).isReadOnly()) .collect(Collectors.toList()); - int i = 1; for (Map.Entry entry : resolvedToolReferences) { progress.setStatus(format(tr("Installing tools ({0}/{1})..."), i, resolvedToolReferences.size())); diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java index 0e6d03794d8..e196d9b414b 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java @@ -374,7 +374,7 @@ public boolean isContributedToolUsed(ContributedPlatform platformToIgnore, Contr if (platformToIgnore.equals(platform)) { continue; } - if (!platform.isInstalled()) { + if (!platform.isInstalled() || platform.isReadOnly()) { continue; } for (ContributedTool requiredTool : platform.getResolvedTools()) { From b695e7ff1ed960859d67c07847f1f2f068a4830b Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 12 Aug 2016 10:27:35 +0200 Subject: [PATCH 7/7] Do not fail if a package_index.json is not present Since we are not bundling a package_index.json anymore, there is no need for the timestamp check with the existing package_index.json. --- .../packages/ContributionsIndexer.java | 19 +++++++++------- .../src/processing/app/BaseNoGui.java | 22 ++----------------- 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java index e196d9b414b..f38d81e4c67 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java @@ -81,17 +81,20 @@ public ContributionsIndexer(File preferencesFolder, File builtInHardwareFolder, } public void parseIndex() throws Exception { - File defaultIndexFile = getIndexFile(Constants.DEFAULT_INDEX_FILE_NAME); + // Read bundled index... File bundledIndexFile = new File(builtInHardwareFolder, Constants.BUNDLED_INDEX_FILE_NAME); + index = parseIndex(bundledIndexFile); - // Check main index signature - if (!PreferencesData.getBoolean("allow_insecure_packages") && !signatureVerifier.isSigned(defaultIndexFile)) { - throw new SignatureVerificationFailedException(Constants.DEFAULT_INDEX_FILE_NAME); - } + // ...and overlay the default index if present + File defaultIndexFile = getIndexFile(Constants.DEFAULT_INDEX_FILE_NAME); + if (defaultIndexFile.exists()) { + // Check main index signature + if (!PreferencesData.getBoolean("allow_insecure_packages") && !signatureVerifier.isSigned(defaultIndexFile)) { + throw new SignatureVerificationFailedException(Constants.DEFAULT_INDEX_FILE_NAME); + } - // Read bundled index and overlay the default index - index = parseIndex(bundledIndexFile); - mergeContributions(parseIndex(defaultIndexFile), defaultIndexFile); + mergeContributions(parseIndex(defaultIndexFile), defaultIndexFile); + } // Set main and bundled indexes as trusted index.getPackages().forEach(pack -> pack.setTrusted(true)); diff --git a/arduino-core/src/processing/app/BaseNoGui.java b/arduino-core/src/processing/app/BaseNoGui.java index 1cef43cbf83..b160605c51e 100644 --- a/arduino-core/src/processing/app/BaseNoGui.java +++ b/arduino-core/src/processing/app/BaseNoGui.java @@ -610,30 +610,12 @@ static public void initLogger() { static public void initPackages() throws Exception { indexer = new ContributionsIndexer(getSettingsFolder(), getHardwareFolder(), getPlatform(), new GPGDetachedSignatureVerifier()); - File indexFile = indexer.getIndexFile("package_index.json"); - File defaultPackageJsonFile = new File(getContentFile("dist"), "package_index.json"); - if (!indexFile.isFile() || (defaultPackageJsonFile.isFile() && defaultPackageJsonFile.lastModified() > indexFile.lastModified())) { - FileUtils.copyFile(defaultPackageJsonFile, indexFile); - } else if (!indexFile.isFile()) { - // Otherwise create an empty packages index - FileOutputStream out = null; - try { - out = new FileOutputStream(indexFile); - out.write("{ \"packages\" : [ ] }".getBytes()); - } finally { - IOUtils.closeQuietly(out); - } - } - - File indexSignatureFile = indexer.getIndexFile("package_index.json.sig"); - File defaultPackageJsonSignatureFile = new File(getContentFile("dist"), "package_index.json.sig"); - if (!indexSignatureFile.isFile() || (defaultPackageJsonSignatureFile.isFile() && defaultPackageJsonSignatureFile.lastModified() > indexSignatureFile.lastModified())) { - FileUtils.copyFile(defaultPackageJsonSignatureFile, indexSignatureFile); - } try { indexer.parseIndex(); } catch (JsonProcessingException | SignatureVerificationFailedException e) { + File indexFile = indexer.getIndexFile(Constants.DEFAULT_INDEX_FILE_NAME); + File indexSignatureFile = indexer.getIndexFile(Constants.DEFAULT_INDEX_FILE_NAME + ".sig"); FileUtils.deleteIfExists(indexFile); FileUtils.deleteIfExists(indexSignatureFile); throw e;