diff --git a/ians-notes.txt b/ians-notes.txt index fe40cc9..6d48423 100644 --- a/ians-notes.txt +++ b/ians-notes.txt @@ -16,3 +16,5 @@ gofmt -s -w shared/*.go cmd/*/*.go For linting: https://github.com/alecthomas/gometalinter + +gometalinter.v2 ./... diff --git a/shared/constants.go b/shared/constants.go index 61cd771..03896b8 100644 --- a/shared/constants.go +++ b/shared/constants.go @@ -39,10 +39,12 @@ package shared // // 1.0.0 // 1.0.1 various bug fixes -// 1.0.2 April 28 2018 linux kernel work, sorting manifest, etc. corresponds with wllvm 1.1.6 - +// 1.0.2 April 28 2018 linux kernel work, sorting bitcode files, etc. +// May 2 2018 handleArchives rewritten to handle multiple occurences of files with the same name. +// corresponds with wllvm 1.1.6 +// const gllvmVersion = "1.0.2" -const gllvmReleaseDate = "April 28 2018" +const gllvmReleaseDate = "May 2 2018" const osDARWIN = "darwin" const osLINUX = "linux" diff --git a/shared/extractor.go b/shared/extractor.go index 888a78f..8cdd755 100644 --- a/shared/extractor.go +++ b/shared/extractor.go @@ -43,6 +43,7 @@ import ( "path/filepath" "runtime" "sort" + "strconv" "strings" ) @@ -57,7 +58,7 @@ type extractionArgs struct { Extractor func(string) []string Verbose bool WriteManifest bool - SortManifest bool + SortBitcodeFiles bool BuildBitcodeArchive bool } @@ -129,7 +130,7 @@ func parseSwitches() (ea extractionArgs) { writeManifestPtr := flag.Bool("m", false, "write the manifest") - sortManifestPtr := flag.Bool("s", false, "sort the manifest") + sortBitcodeFilesPtr := flag.Bool("s", false, "sort the bitcode files") buildBitcodeArchive := flag.Bool("b", false, "build a bitcode module(FIXME? should this be archive)") @@ -143,7 +144,7 @@ func parseSwitches() (ea extractionArgs) { ea.Verbose = *verbosePtr ea.WriteManifest = *writeManifestPtr - ea.SortManifest = *sortManifestPtr + ea.SortBitcodeFiles = *sortBitcodeFilesPtr ea.BuildBitcodeArchive = *buildBitcodeArchive if *archiverNamePtr != "" { @@ -208,24 +209,20 @@ func handleExecutable(ea extractionArgs) { for i, artPath := range artifactPaths { filesToLink[i] = resolveBitcodePath(artPath) } - extractTimeLinkFiles(ea, filesToLink) + + // Sort the bitcode files + if ea.SortBitcodeFiles { + LogWarning("Sorting bitcode files.") + sort.Strings(filesToLink) + sort.Strings(artifactPaths) + } // Write manifest if ea.WriteManifest { writeManifest(ea, filesToLink, artifactPaths) } -} -func extractFromThinArchive(inputFile string) (contents []string) { - var arArgs []string - arArgs = append(arArgs, "-t") - arArgs = append(arArgs, inputFile) - output, err := runCmd("ar", arArgs) - if err != nil { - LogFatal("Failed to extract from thin archive %s because: %v.\n", inputFile, err) - } - contents = strings.Split(output, "\n") - return + extractTimeLinkFiles(ea, filesToLink) } func handleThinArchive(ea extractionArgs) { @@ -235,7 +232,7 @@ func handleThinArchive(ea extractionArgs) { var objectFiles []string var bcFiles []string - objectFiles = extractFromThinArchive(ea.InputFile) + objectFiles = listArchiveFiles(ea.InputFile) LogInfo("handleThinArchive: extractionArgs = %v\nobjectFiles = %v\n", ea, objectFiles) @@ -260,6 +257,14 @@ func handleThinArchive(ea extractionArgs) { LogInfo("len(bcFiles) = %v\n", len(bcFiles)) if len(bcFiles) > 0 { + + // Sort the bitcode files + if ea.SortBitcodeFiles { + LogWarning("Sorting bitcode files.") + sort.Strings(bcFiles) + sort.Strings(artifactFiles) + } + // Build archive if ea.BuildBitcodeArchive { extractTimeLinkFiles(ea, bcFiles) @@ -277,12 +282,68 @@ func handleThinArchive(ea extractionArgs) { } +func listArchiveFiles(inputFile string) (contents []string) { + var arArgs []string + arArgs = append(arArgs, "-t") + arArgs = append(arArgs, inputFile) + output, err := runCmd("ar", arArgs) + if err != nil { + LogWarning("ar command: ar %v", arArgs) + LogFatal("Failed to extract contents from archive %s because: %v.\n", inputFile, err) + } + contents = strings.Split(output, "\n") + return +} + +func extractFile(archive string, filename string, instance int) bool { + var arArgs []string + arArgs = append(arArgs, "xN") + arArgs = append(arArgs, strconv.Itoa(instance)) + arArgs = append(arArgs, archive) + arArgs = append(arArgs, filename) + _, err := runCmd("ar", arArgs) + if err != nil { + LogWarning("Failed to extract instance %v of %v from archive %s because: %v.\n", instance, filename, archive, err) + return false + } + return true +} + +func fetchTOC(inputFile string) map[string]int { + toc := make(map[string]int) + + contents := listArchiveFiles(inputFile) + + for _, item := range contents { + if item != "" { + toc[item]++ + } + } + return toc +} + +//handleArchive processes a archive, and creates either a bitcode archive, or a module, depending on the flags used. +// +// Archives are strange beasts. handleArchive processes the archive by: +// +// 1. first creating a table of contents of the archive, which maps file names (in the archive) to the number of +// times a file with that name is stored in the archive. +// +// 2. for each OCCURENCE of a file (name and count) it extracts the section from the object file, and adds the +// bitcode paths to the bitcode list. +// +// 3. it then either links all these bitcode files together using llvm-link, or else is creates a bitcode +// archive using llvm-ar +// +//iam: 5/1/2018 func handleArchive(ea extractionArgs) { // List bitcode files to link var bcFiles []string var artifactFiles []string - LogInfo("handleArchive: extractionArgs = %v\n", ea) + inputFile, _ := filepath.Abs(ea.InputFile) + + LogWarning("handleArchive: extractionArgs = %v\n", ea) // Create tmp dir tmpDirName, err := ioutil.TempDir("", "gllvm") @@ -291,45 +352,55 @@ func handleArchive(ea extractionArgs) { } defer CheckDefer(func() error { return os.RemoveAll(tmpDirName) }) - // Extract objects to tmpDir - arArgs := ea.ArArgs - inputAbsPath, _ := filepath.Abs(ea.InputFile) - arArgs = append(arArgs, inputAbsPath) - - LogInfo("handleArchive: executing ar %v %v\n", arArgs, tmpDirName) - - success, err := execCmd("ar", arArgs, tmpDirName) - if !success { - LogFatal("Failed to extract object files from %s to %s because: %v.\n", ea.InputFile, tmpDirName, err) + homeDir, err := os.Getwd() + if err != nil { + LogFatal("Could not ascertain our whereabouts: %v", err) } - // Define object file handling closure - var walkHandlingFunc = func(path string, info os.FileInfo, err error) error { - if err == nil && !info.IsDir() { - fileType := getFileType(path) - if fileType == ea.ObjectTypeInArchive { - artifactPaths := ea.Extractor(path) - for _, artPath := range artifactPaths { - bcPath := resolveBitcodePath(artPath) + err = os.Chdir(tmpDirName) + if err != nil { + LogFatal("Could not cd to %v because: %v", tmpDirName, err) + } + + //1. fetch the Table of Contents + toc := fetchTOC(inputFile) + + LogDebug("Table of Contents of %v:\n%v\n", inputFile, toc) + + for obj, instance := range toc { + for i := 1; i <= instance; i++ { + + if obj != "" && extractFile(inputFile, obj, i) { + + artifacts := ea.Extractor(obj) + LogInfo("\t%v\n", artifacts) + artifactFiles = append(artifactFiles, artifacts...) + for _, bc := range artifacts { + bcPath := resolveBitcodePath(bc) if bcPath != "" { bcFiles = append(bcFiles, bcPath) } } - artifactFiles = append(artifactFiles, artifactPaths...) } } - return nil } - // Handle object files - err = filepath.Walk(tmpDirName, walkHandlingFunc) + err = os.Chdir(homeDir) if err != nil { - LogFatal("handleArchive: walking %v failed with %v\n", tmpDirName, err) + LogFatal("Could not cd to %v because: %v", homeDir, err) } LogDebug("handleArchive: walked %v\nartifactFiles:\n%v\nbcFiles:\n%v\n", tmpDirName, artifactFiles, bcFiles) if len(bcFiles) > 0 { + + // Sort the bitcode files + if ea.SortBitcodeFiles { + LogWarning("Sorting bitcode files.") + sort.Strings(bcFiles) + sort.Strings(artifactFiles) + } + // Build archive if ea.BuildBitcodeArchive { extractTimeLinkFiles(ea, bcFiles) @@ -368,7 +439,7 @@ func archiveBcFiles(ea extractionArgs, bcFiles []string) { LogFatal("There was an error creating the bitcode archive: %v.\n", err) } } - LogInfo("Built bitcode archive: %s.", ea.OutputFile) + LogWarning("Built bitcode archive: %s.", ea.OutputFile) } func extractTimeLinkFiles(ea extractionArgs, filesToLink []string) { @@ -444,16 +515,19 @@ func resolveBitcodePath(bcPath string) string { func writeManifest(ea extractionArgs, bcFiles []string, artifactFiles []string) { manifestFilename := ea.OutputFile + ".llvm.manifest" - if ea.SortManifest { - LogWarning("Manifest sorting %s.", manifestFilename) - sort.Strings(bcFiles) - sort.Strings(artifactFiles) + //only go into the gory details if we have a store around. + if LLVMBitcodeStorePath != "" { + section1 := "Physical location of extracted files:\n" + strings.Join(bcFiles, "\n") + "\n\n" + section2 := "Build-time location of extracted files:\n" + strings.Join(artifactFiles, "\n") + contents := []byte(section1 + section2) + if err := ioutil.WriteFile(manifestFilename, contents, 0644); err != nil { + LogFatal("There was an error while writing the manifest file: ", err) + } + } else { + contents := []byte("\n" + strings.Join(bcFiles, "\n") + "\n") + if err := ioutil.WriteFile(manifestFilename, contents, 0644); err != nil { + LogFatal("There was an error while writing the manifest file: ", err) + } } - section1 := "Physical location of extracted files:\n" + strings.Join(bcFiles, "\n") + "\n\n" - section2 := "Build-time location of extracted files:\n" + strings.Join(artifactFiles, "\n") - contents := []byte(section1 + section2) - if err := ioutil.WriteFile(manifestFilename, contents, 0644); err != nil { - LogFatal("There was an error while writing the manifest file: ", err) - } - LogInfo("Manifest file written to %s.", manifestFilename) + LogWarning("Manifest file written to %s.", manifestFilename) }