Handles archives better. Flag for sorting bitcode files. Linux Kernel work.

ians-notes.txt

@@ -16,3 +16,5 @@ gofmt -s -w shared/*.go cmd/*/*.go
 For linting:
 
 https://github.com/alecthomas/gometalinter
+
+gometalinter.v2 ./...

shared/constants.go

@@ -39,10 +39,12 @@ package shared
 //
 //  1.0.0
 //  1.0.1                 various bug fixes
-//  1.0.2 April 28 2018   linux kernel work, sorting manifest, etc. corresponds with wllvm 1.1.6
-
+//  1.0.2 April 28 2018   linux kernel work, sorting bitcode files, etc.
+//        May 2 2018      handleArchives rewritten to handle multiple occurences of files with the same name.
+//                        corresponds with wllvm 1.1.6
+//
 const gllvmVersion = "1.0.2"
-const gllvmReleaseDate = "April 28 2018"
+const gllvmReleaseDate = "May 2 2018"
 
 const osDARWIN = "darwin"
 const osLINUX = "linux"

shared/extractor.go

@@ -43,6 +43,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"sort"
+	"strconv"
 	"strings"
 )
 
@@ -57,7 +58,7 @@ type extractionArgs struct {
 	Extractor           func(string) []string
 	Verbose             bool
 	WriteManifest       bool
-	SortManifest        bool
+	SortBitcodeFiles    bool
 	BuildBitcodeArchive bool
 }
 
@@ -129,7 +130,7 @@ func parseSwitches() (ea extractionArgs) {
 
 	writeManifestPtr := flag.Bool("m", false, "write the manifest")
 
-	sortManifestPtr := flag.Bool("s", false, "sort the manifest")
+	sortBitcodeFilesPtr := flag.Bool("s", false, "sort the bitcode files")
 
 	buildBitcodeArchive := flag.Bool("b", false, "build a bitcode module(FIXME? should this be archive)")
 
@@ -143,7 +144,7 @@ func parseSwitches() (ea extractionArgs) {
 
 	ea.Verbose = *verbosePtr
 	ea.WriteManifest = *writeManifestPtr
-	ea.SortManifest = *sortManifestPtr
+	ea.SortBitcodeFiles = *sortBitcodeFilesPtr
 	ea.BuildBitcodeArchive = *buildBitcodeArchive
 
 	if *archiverNamePtr != "" {
@@ -208,24 +209,20 @@ func handleExecutable(ea extractionArgs) {
 	for i, artPath := range artifactPaths {
 		filesToLink[i] = resolveBitcodePath(artPath)
 	}
-	extractTimeLinkFiles(ea, filesToLink)
+
+	// Sort the bitcode files
+	if ea.SortBitcodeFiles {
+		LogWarning("Sorting bitcode files.")
+		sort.Strings(filesToLink)
+		sort.Strings(artifactPaths)
+	}
 
 	// Write manifest
 	if ea.WriteManifest {
 		writeManifest(ea, filesToLink, artifactPaths)
 	}
-}
 
-func extractFromThinArchive(inputFile string) (contents []string) {
-	var arArgs []string
-	arArgs = append(arArgs, "-t")
-	arArgs = append(arArgs, inputFile)
-	output, err := runCmd("ar", arArgs)
-	if err != nil {
-		LogFatal("Failed to extract from thin archive %s because: %v.\n", inputFile, err)
-	}
-	contents = strings.Split(output, "\n")
-	return
+	extractTimeLinkFiles(ea, filesToLink)
 }
 
 func handleThinArchive(ea extractionArgs) {
@@ -235,7 +232,7 @@ func handleThinArchive(ea extractionArgs) {
 	var objectFiles []string
 	var bcFiles []string
 
-	objectFiles = extractFromThinArchive(ea.InputFile)
+	objectFiles = listArchiveFiles(ea.InputFile)
 
 	LogInfo("handleThinArchive: extractionArgs = %v\nobjectFiles = %v\n", ea, objectFiles)
 
@@ -260,6 +257,14 @@ func handleThinArchive(ea extractionArgs) {
 	LogInfo("len(bcFiles) = %v\n", len(bcFiles))
 
 	if len(bcFiles) > 0 {
+
+		// Sort the bitcode files
+		if ea.SortBitcodeFiles {
+			LogWarning("Sorting bitcode files.")
+			sort.Strings(bcFiles)
+			sort.Strings(artifactFiles)
+		}
+
 		// Build archive
 		if ea.BuildBitcodeArchive {
 			extractTimeLinkFiles(ea, bcFiles)
@@ -277,12 +282,68 @@ func handleThinArchive(ea extractionArgs) {
 
 }
 
+func listArchiveFiles(inputFile string) (contents []string) {
+	var arArgs []string
+	arArgs = append(arArgs, "-t")
+	arArgs = append(arArgs, inputFile)
+	output, err := runCmd("ar", arArgs)
+	if err != nil {
+		LogWarning("ar command: ar %v", arArgs)
+		LogFatal("Failed to extract contents from archive %s because: %v.\n", inputFile, err)
+	}
+	contents = strings.Split(output, "\n")
+	return
+}
+
+func extractFile(archive string, filename string, instance int) bool {
+	var arArgs []string
+	arArgs = append(arArgs, "xN")
+	arArgs = append(arArgs, strconv.Itoa(instance))
+	arArgs = append(arArgs, archive)
+	arArgs = append(arArgs, filename)
+	_, err := runCmd("ar", arArgs)
+	if err != nil {
+		LogWarning("Failed to extract instance %v of %v from archive %s because: %v.\n", instance, filename, archive, err)
+		return false
+	}
+	return true
+}
+
+func fetchTOC(inputFile string) map[string]int {
+	toc := make(map[string]int)
+
+	contents := listArchiveFiles(inputFile)
+
+	for _, item := range contents {
+		if item != "" {
+			toc[item]++
+		}
+	}
+	return toc
+}
+
+//handleArchive processes a archive, and creates either a bitcode archive, or a module, depending on the flags used.
+//
+//    Archives are strange beasts. handleArchive processes the archive by:
+//
+//      1. first creating a table of contents of the archive, which maps file names (in the archive) to the number of
+//    times a file with that name is stored in the archive.
+//
+//      2. for each OCCURENCE of a file (name and count) it extracts the section from the object file, and adds the
+//    bitcode paths to the bitcode list.
+//
+//      3. it then either links all these bitcode files together using llvm-link,  or else is creates a bitcode
+//    archive using llvm-ar
+//
+//iam: 5/1/2018
 func handleArchive(ea extractionArgs) {
 	// List bitcode files to link
 	var bcFiles []string
 	var artifactFiles []string
 
-	LogInfo("handleArchive: extractionArgs = %v\n", ea)
+	inputFile, _ := filepath.Abs(ea.InputFile)
+
+	LogWarning("handleArchive: extractionArgs = %v\n", ea)
 
 	// Create tmp dir
 	tmpDirName, err := ioutil.TempDir("", "gllvm")
@@ -291,45 +352,55 @@ func handleArchive(ea extractionArgs) {
 	}
 	defer CheckDefer(func() error { return os.RemoveAll(tmpDirName) })
 
-	// Extract objects to tmpDir
-	arArgs := ea.ArArgs
-	inputAbsPath, _ := filepath.Abs(ea.InputFile)
-	arArgs = append(arArgs, inputAbsPath)
-
-	LogInfo("handleArchive: executing  ar %v %v\n", arArgs, tmpDirName)
-
-	success, err := execCmd("ar", arArgs, tmpDirName)
-	if !success {
-		LogFatal("Failed to extract object files from %s to %s because: %v.\n", ea.InputFile, tmpDirName, err)
+	homeDir, err := os.Getwd()
+	if err != nil {
+		LogFatal("Could not ascertain our whereabouts: %v", err)
 	}
 
-	// Define object file handling closure
-	var walkHandlingFunc = func(path string, info os.FileInfo, err error) error {
-		if err == nil && !info.IsDir() {
-			fileType := getFileType(path)
-			if fileType == ea.ObjectTypeInArchive {
-				artifactPaths := ea.Extractor(path)
-				for _, artPath := range artifactPaths {
-					bcPath := resolveBitcodePath(artPath)
+	err = os.Chdir(tmpDirName)
+	if err != nil {
+		LogFatal("Could not cd to %v because: %v", tmpDirName, err)
+	}
+
+	//1. fetch the Table of Contents
+	toc := fetchTOC(inputFile)
+
+	LogDebug("Table of Contents of %v:\n%v\n", inputFile, toc)
+
+	for obj, instance := range toc {
+		for i := 1; i <= instance; i++ {
+
+			if obj != "" && extractFile(inputFile, obj, i) {
+
+				artifacts := ea.Extractor(obj)
+				LogInfo("\t%v\n", artifacts)
+				artifactFiles = append(artifactFiles, artifacts...)
+				for _, bc := range artifacts {
+					bcPath := resolveBitcodePath(bc)
 					if bcPath != "" {
 						bcFiles = append(bcFiles, bcPath)
 					}
 				}
-				artifactFiles = append(artifactFiles, artifactPaths...)
 			}
 		}
-		return nil
 	}
 
-	// Handle object files
-	err = filepath.Walk(tmpDirName, walkHandlingFunc)
+	err = os.Chdir(homeDir)
 	if err != nil {
-		LogFatal("handleArchive: walking %v failed with %v\n", tmpDirName, err)
+		LogFatal("Could not cd to %v because: %v", homeDir, err)
 	}
 
 	LogDebug("handleArchive: walked %v\nartifactFiles:\n%v\nbcFiles:\n%v\n", tmpDirName, artifactFiles, bcFiles)
 
 	if len(bcFiles) > 0 {
+
+		// Sort the bitcode files
+		if ea.SortBitcodeFiles {
+			LogWarning("Sorting bitcode files.")
+			sort.Strings(bcFiles)
+			sort.Strings(artifactFiles)
+		}
+
 		// Build archive
 		if ea.BuildBitcodeArchive {
 			extractTimeLinkFiles(ea, bcFiles)
@@ -368,7 +439,7 @@ func archiveBcFiles(ea extractionArgs, bcFiles []string) {
 			LogFatal("There was an error creating the bitcode archive: %v.\n", err)
 		}
 	}
-	LogInfo("Built bitcode archive: %s.", ea.OutputFile)
+	LogWarning("Built bitcode archive: %s.", ea.OutputFile)
 }
 
 func extractTimeLinkFiles(ea extractionArgs, filesToLink []string) {
@@ -444,16 +515,19 @@ func resolveBitcodePath(bcPath string) string {
 
 func writeManifest(ea extractionArgs, bcFiles []string, artifactFiles []string) {
 	manifestFilename := ea.OutputFile + ".llvm.manifest"
-	if ea.SortManifest {
-		LogWarning("Manifest sorting %s.", manifestFilename)
-		sort.Strings(bcFiles)
-		sort.Strings(artifactFiles)
+	//only go into the gory details if we have a store around.
+	if LLVMBitcodeStorePath != "" {
+		section1 := "Physical location of extracted files:\n" + strings.Join(bcFiles, "\n") + "\n\n"
+		section2 := "Build-time location of extracted files:\n" + strings.Join(artifactFiles, "\n")
+		contents := []byte(section1 + section2)
+		if err := ioutil.WriteFile(manifestFilename, contents, 0644); err != nil {
+			LogFatal("There was an error while writing the manifest file: ", err)
+		}
+	} else {
+		contents := []byte("\n" + strings.Join(bcFiles, "\n") + "\n")
+		if err := ioutil.WriteFile(manifestFilename, contents, 0644); err != nil {
+			LogFatal("There was an error while writing the manifest file: ", err)
+		}
 	}
-	section1 := "Physical location of extracted files:\n" + strings.Join(bcFiles, "\n") + "\n\n"
-	section2 := "Build-time location of extracted files:\n" + strings.Join(artifactFiles, "\n")
-	contents := []byte(section1 + section2)
-	if err := ioutil.WriteFile(manifestFilename, contents, 0644); err != nil {
-		LogFatal("There was an error while writing the manifest file: ", err)
-	}
-	LogInfo("Manifest file written to %s.", manifestFilename)
+	LogWarning("Manifest file written to %s.", manifestFilename)
 }