Include detection refactor

src/arduino.cc/builder/add_additional_entries_to_context.go

@@ -69,7 +69,6 @@ func (s *AddAdditionalEntriesToContext) Run(ctx *types.Context) error {
 	}
 
 	ctx.CollectedSourceFiles = &types.UniqueStringQueue{}
-	ctx.FoldersWithSourceFiles = &types.UniqueSourceFolderQueue{}
 
 	ctx.LibrariesResolutionResults = make(map[string]types.LibraryResolutionResult)
 	ctx.HardwareRewriteResults = make(map[*types.Platform][]types.PlatforKeyRewrite)

src/arduino.cc/builder/container_find_includes.go

@@ -42,52 +42,30 @@ import (
 type ContainerFindIncludes struct{}
 
 func (s *ContainerFindIncludes) Run(ctx *types.Context) error {
-	err := runCommand(ctx, &IncludesToIncludeFolders{})
-	if err != nil {
-		return i18n.WrapError(err)
+	ctx.IncludeFolders = append(ctx.IncludeFolders, ctx.BuildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH])
+	if ctx.BuildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH] != constants.EMPTY_STRING {
+		ctx.IncludeFolders = append(ctx.IncludeFolders, ctx.BuildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH])
 	}
 
 	sketchBuildPath := ctx.SketchBuildPath
 	sketch := ctx.Sketch
-	err = findIncludesUntilDone(ctx, filepath.Join(sketchBuildPath, filepath.Base(sketch.MainFile.Name)+".cpp"))
-	if err != nil {
-		return i18n.WrapError(err)
-	}
+	ctx.CollectedSourceFiles.Push(filepath.Join(sketchBuildPath, filepath.Base(sketch.MainFile.Name)+".cpp"))
 
-	foldersWithSources := ctx.FoldersWithSourceFiles
-	foldersWithSources.Push(types.SourceFolder{Folder: ctx.SketchBuildPath, Recurse: false})
+	sourceFilePaths := ctx.CollectedSourceFiles
+	queueSourceFilesFromFolder(sourceFilePaths, ctx.SketchBuildPath, /* recurse */ false)
 	srcSubfolderPath := filepath.Join(ctx.SketchBuildPath, constants.SKETCH_FOLDER_SRC)
 	if info, err := os.Stat(srcSubfolderPath); err == nil && info.IsDir() {
-		foldersWithSources.Push(types.SourceFolder{Folder: srcSubfolderPath, Recurse: true})
-	}
-	if len(ctx.ImportedLibraries) > 0 {
-		for _, library := range ctx.ImportedLibraries {
-			sourceFolders := types.LibraryToSourceFolder(library)
-			for _, sourceFolder := range sourceFolders {
-				foldersWithSources.Push(sourceFolder)
-			}
-		}
-	}
-
-	err = runCommand(ctx, &CollectAllSourceFilesFromFoldersWithSources{})
-	if err != nil {
-		return i18n.WrapError(err)
+		queueSourceFilesFromFolder(sourceFilePaths, srcSubfolderPath, /* recurse */ true)
 	}
 
-	sourceFilePaths := ctx.CollectedSourceFiles
-
 	for !sourceFilePaths.Empty() {
-		err = findIncludesUntilDone(ctx, sourceFilePaths.Pop().(string))
-		if err != nil {
-			return i18n.WrapError(err)
-		}
-		err := runCommand(ctx, &CollectAllSourceFilesFromFoldersWithSources{})
+		err := findIncludesUntilDone(ctx, sourceFilePaths.Pop().(string))
 		if err != nil {
 			return i18n.WrapError(err)
 		}
 	}
 
-	err = runCommand(ctx, &FailIfImportedLibraryIsWrong{})
+	err := runCommand(ctx, &FailIfImportedLibraryIsWrong{})
 	if err != nil {
 		return i18n.WrapError(err)
 	}
@@ -106,28 +84,53 @@ func runCommand(ctx *types.Context, command types.Command) error {
 
 func findIncludesUntilDone(ctx *types.Context, sourceFilePath string) error {
 	targetFilePath := utils.NULLFile()
-	importedLibraries := ctx.ImportedLibraries
-	done := false
-	for !done {
+	for {
 		commands := []types.Command{
 			&GCCPreprocRunnerForDiscoveringIncludes{SourceFilePath: sourceFilePath, TargetFilePath: targetFilePath},
 			&IncludesFinderWithRegExp{Source: &ctx.SourceGccMinusE},
-			&IncludesToIncludeFolders{},
 		}
 		for _, command := range commands {
 			err := runCommand(ctx, command)
 			if err != nil {
 				return i18n.WrapError(err)
 			}
 		}
-		if len(ctx.IncludesJustFound) == 0 {
-			done = true
-		} else if len(ctx.ImportedLibraries) == len(importedLibraries) {
+		if ctx.IncludeJustFound == "" {
+			// No missing includes found, we're done
+			return nil
+		}
+
+		library := ResolveLibrary(ctx, ctx.IncludeJustFound)
+		if library == nil {
+			// Library could not be resolved, show error
 			err := runCommand(ctx, &GCCPreprocRunner{TargetFileName: constants.FILE_CTAGS_TARGET_FOR_GCC_MINUS_E})
 			return i18n.WrapError(err)
 		}
-		importedLibraries = ctx.ImportedLibraries
-		ctx.IncludesJustFound = []string{}
+
+		// Add this library to the list of libraries, the
+		// include path and queue its source files for further
+		// include scanning
+		ctx.ImportedLibraries = append(ctx.ImportedLibraries, library)
+		ctx.IncludeFolders = append(ctx.IncludeFolders, library.SrcFolder)
+		sourceFolders := types.LibraryToSourceFolder(library)
+		for _, sourceFolder := range sourceFolders {
+			queueSourceFilesFromFolder(ctx.CollectedSourceFiles, sourceFolder.Folder, sourceFolder.Recurse)
+		}
 	}
+}
+
+func queueSourceFilesFromFolder(queue *types.UniqueStringQueue, folder string, recurse bool) error {
+	extensions := func(ext string) bool { return ADDITIONAL_FILE_VALID_EXTENSIONS_NO_HEADERS[ext] }
+
+	filePaths := []string{}
+	err := utils.FindFilesInFolder(&filePaths, folder, extensions, recurse)
+	if err != nil {
+		return i18n.WrapError(err)
+	}
+
+	for _, filePath := range filePaths {
+		queue.Push(filePath)
+	}
+
 	return nil
 }

src/arduino.cc/builder/includes_finder_with_regexp.go

@@ -31,7 +31,6 @@ package builder
 
 import (
 	"arduino.cc/builder/types"
-	"arduino.cc/builder/utils"
 	"regexp"
 	"strings"
 )
@@ -45,24 +44,17 @@ type IncludesFinderWithRegExp struct {
 func (s *IncludesFinderWithRegExp) Run(ctx *types.Context) error {
 	source := *s.Source
 
-	matches := INCLUDE_REGEXP.FindAllStringSubmatch(source, -1)
-	includes := []string{}
-	for _, match := range matches {
-		includes = append(includes, strings.TrimSpace(match[1]))
-	}
-	if len(includes) == 0 {
-		include := findIncludesForOldCompilers(source)
-		if include != "" {
-			includes = append(includes, include)
-		}
+	match := INCLUDE_REGEXP.FindStringSubmatch(source)
+	if match != nil {
+		ctx.IncludeJustFound = strings.TrimSpace(match[1])
+	} else {
+		ctx.IncludeJustFound = findIncludeForOldCompilers(source)
 	}
 
-	ctx.IncludesJustFound = includes
-	ctx.Includes = utils.AppendIfNotPresent(ctx.Includes, includes...)
 	return nil
 }
 
-func findIncludesForOldCompilers(source string) string {
+func findIncludeForOldCompilers(source string) string {
 	lines := strings.Split(source, "\n")
 	for _, line := range lines {
 		splittedLine := strings.Split(line, ":")

src/arduino.cc/builder/includes_to_include_folders.go → src/arduino.cc/builder/resolve_library.go

@@ -34,92 +34,34 @@ import (
 	"strings"
 
 	"arduino.cc/builder/constants"
-	"arduino.cc/builder/i18n"
 	"arduino.cc/builder/types"
 	"arduino.cc/builder/utils"
-	"arduino.cc/properties"
 )
 
-type IncludesToIncludeFolders struct{}
-
-func (s *IncludesToIncludeFolders) Run(ctx *types.Context) error {
-	includes := ctx.Includes
+func ResolveLibrary(ctx *types.Context, header string) *types.Library {
 	headerToLibraries := ctx.HeaderToLibraries
-
-	platform := ctx.TargetPlatform
-	actualPlatform := ctx.ActualPlatform
+	platforms := []*types.Platform{ctx.ActualPlatform, ctx.TargetPlatform}
 	libraryResolutionResults := ctx.LibrariesResolutionResults
 	importedLibraries := ctx.ImportedLibraries
 
-	newlyImportedLibraries, err := resolveLibraries(includes, headerToLibraries, importedLibraries, []*types.Platform{actualPlatform, platform}, libraryResolutionResults)
-	if err != nil {
-		return i18n.WrapError(err)
-	}
-
-	foldersWithSources := ctx.FoldersWithSourceFiles
-
-	for _, newlyImportedLibrary := range newlyImportedLibraries {
-		if !sliceContainsLibrary(importedLibraries, newlyImportedLibrary) {
-			importedLibraries = append(importedLibraries, newlyImportedLibrary)
-			sourceFolders := types.LibraryToSourceFolder(newlyImportedLibrary)
-			for _, sourceFolder := range sourceFolders {
-				foldersWithSources.Push(sourceFolder)
-			}
-		}
-	}
-
-	ctx.ImportedLibraries = importedLibraries
-	ctx.IncludeFolders = resolveIncludeFolders(newlyImportedLibraries, ctx.BuildProperties, ctx.Verbose)
-
-	return nil
-}
-
-func resolveIncludeFolders(importedLibraries []*types.Library, buildProperties properties.Map, verbose bool) []string {
-	var includeFolders []string
-	includeFolders = append(includeFolders, buildProperties[constants.BUILD_PROPERTIES_BUILD_CORE_PATH])
-	if buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH] != constants.EMPTY_STRING {
-		includeFolders = append(includeFolders, buildProperties[constants.BUILD_PROPERTIES_BUILD_VARIANT_PATH])
-	}
-
-	for _, library := range importedLibraries {
-		includeFolders = append(includeFolders, library.SrcFolder)
-	}
-
-	return includeFolders
-}
-
-//FIXME it's also resolving previously resolved libraries
-func resolveLibraries(includes []string, headerToLibraries map[string][]*types.Library, importedLibraries []*types.Library, platforms []*types.Platform, libraryResolutionResults map[string]types.LibraryResolutionResult) ([]*types.Library, error) {
 	markImportedLibrary := make(map[*types.Library]bool)
 	for _, library := range importedLibraries {
 		markImportedLibrary[library] = true
 	}
-	for _, header := range includes {
-		resolveLibrary(header, headerToLibraries, markImportedLibrary, platforms, libraryResolutionResults)
-	}
-
-	var newlyImportedLibraries []*types.Library
-	for library, _ := range markImportedLibrary {
-		newlyImportedLibraries = append(newlyImportedLibraries, library)
-	}
 
-	return newlyImportedLibraries, nil
-}
-
-func resolveLibrary(header string, headerToLibraries map[string][]*types.Library, markImportedLibrary map[*types.Library]bool, platforms []*types.Platform, libraryResolutionResults map[string]types.LibraryResolutionResult) {
 	libraries := append([]*types.Library{}, headerToLibraries[header]...)
 
 	if libraries == nil || len(libraries) == 0 {
-		return
+		return nil
 	}
 
 	if len(libraries) == 1 {
 		markImportedLibrary[libraries[0]] = true
-		return
+		return libraries[0]
 	}
 
 	if markImportedLibraryContainsOneOfCandidates(markImportedLibrary, libraries) {
-		return
+		return nil
 	}
 
 	reverse(libraries)
@@ -151,6 +93,7 @@ func resolveLibrary(header string, headerToLibraries map[string][]*types.Library
 	libraryResolutionResults[header] = types.LibraryResolutionResult{Library: library, NotUsedLibraries: filterOutLibraryFrom(libraries, library)}
 
 	markImportedLibrary[library] = true
+	return library
 }
 
 //facepalm. sort.Reverse needs an Interface that implements Len/Less/Swap. It's a slice! What else for reversing it?!?

src/arduino.cc/builder/test/add_additional_entries_to_context_test.go

@@ -52,7 +52,6 @@ func TestAddAdditionalEntriesToContextNoBuildPath(t *testing.T) {
 	require.NotNil(t, ctx.WarningsLevel)
 
 	require.True(t, ctx.CollectedSourceFiles.Empty())
-	require.True(t, ctx.FoldersWithSourceFiles.Empty())
 
 	require.Equal(t, 0, len(ctx.LibrariesResolutionResults))
 }
@@ -72,7 +71,6 @@ func TestAddAdditionalEntriesToContextWithBuildPath(t *testing.T) {
 	require.NotNil(t, ctx.WarningsLevel)
 
 	require.True(t, ctx.CollectedSourceFiles.Empty())
-	require.True(t, ctx.FoldersWithSourceFiles.Empty())
 
 	require.Equal(t, 0, len(ctx.LibrariesResolutionResults))
 }