From dfb22031a1b0914677031e120541e2eb07fda32d Mon Sep 17 00:00:00 2001 From: "Ian A. Mason" Date: Sun, 11 Oct 2020 17:22:33 -0700 Subject: [PATCH] Attempt at making the lto tweaks work properly, and adding the LTO_LINKING_FLAGS environment example. --- cmd/gparse/main.go | 49 ++++++++++++++ examples/lto/Makefile | 21 ++++++ examples/lto/a.c | 24 +++++++ examples/lto/a.h | 4 ++ examples/lto/main.c | 11 ++++ shared/compiler.go | 76 ++++++++++++++++++---- shared/constants.go | 6 +- shared/environment.go | 9 ++- shared/parser.go | 146 +++++++++++++++++++++++++++++------------- 9 files changed, 286 insertions(+), 60 deletions(-) create mode 100644 cmd/gparse/main.go create mode 100644 examples/lto/Makefile create mode 100644 examples/lto/a.c create mode 100644 examples/lto/a.h create mode 100644 examples/lto/main.c diff --git a/cmd/gparse/main.go b/cmd/gparse/main.go new file mode 100644 index 0000000..ea0b3f8 --- /dev/null +++ b/cmd/gparse/main.go @@ -0,0 +1,49 @@ +// +// OCCAM +// +// Copyright (c) 2017, SRI International +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of SRI International nor the names of its contributors may +// be used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +package main + +import ( + "fmt" + "github.com/SRI-CSL/gllvm/shared" + "os" +) + +func main() { + // Parse command line + args := os.Args + args = args[1:] + parsed := shared.Parse(args) + // Print out the result + fmt.Printf("Parsed: %v", &parsed) +} diff --git a/examples/lto/Makefile b/examples/lto/Makefile new file mode 100644 index 0000000..948162a --- /dev/null +++ b/examples/lto/Makefile @@ -0,0 +1,21 @@ + + +all: main + +main: main.o + ${CC} -flto a.o main.o -o main + +main.o: main.c a.o + ${CC} -c main.c -o main.o + +a.o: a.c a.h + ${CC} -flto -c a.c -o a.o + +plain: main.c a.c a.h + ${CC} main.c a.c -o plain + + +clean: + rm -f *.o main plain + + diff --git a/examples/lto/a.c b/examples/lto/a.c new file mode 100644 index 0000000..d933882 --- /dev/null +++ b/examples/lto/a.c @@ -0,0 +1,24 @@ +//https://llvm.org/docs/LinkTimeOptimization.html +#include "a.h" + +static signed int i = 0; + +void foo2(void) { + i = -1; +} + +static int foo3() { + foo4(); + return 10; +} + +int foo1(void) { + int data = 0; + + if (i < 0) + data = foo3(); + + data = data + 42; + return data; +} + diff --git a/examples/lto/a.h b/examples/lto/a.h new file mode 100644 index 0000000..f7dd151 --- /dev/null +++ b/examples/lto/a.h @@ -0,0 +1,4 @@ +//https://llvm.org/docs/LinkTimeOptimization.html +extern int foo1(void); +extern void foo2(void); +extern void foo4(void); diff --git a/examples/lto/main.c b/examples/lto/main.c new file mode 100644 index 0000000..8f1760e --- /dev/null +++ b/examples/lto/main.c @@ -0,0 +1,11 @@ +//https://llvm.org/docs/LinkTimeOptimization.html +#include +#include "a.h" + +void foo4(void) { + printf("Hi\n"); +} + +int main() { + return foo1(); +} diff --git a/shared/compiler.go b/shared/compiler.go index d819bfd..2c17c7b 100644 --- a/shared/compiler.go +++ b/shared/compiler.go @@ -53,19 +53,21 @@ type bitcodeToObjectLink struct { func Compile(args []string, compiler string) (exitCode int) { exitCode = 0 - //in the configureOnly case we have to know the exit code of the compile - //because that is how configure figures out what it can and cannot do. + // in the configureOnly case we have to know the exit code of the compile + // because that is how configure figures out what it can and cannot do. ok := true compilerExecName := GetCompilerExecName(compiler) - pr := parse(args) + pr := Parse(args) var wg sync.WaitGroup - // If configure only or print only are set, just execute the compiler - if skipBitcodeGeneration(pr) { + LogDebug("Compile using parsed arguments:%v\n", &pr) + + // If configure only, emit-llvm, flto, or print only are set, just execute the compiler + if pr.skipBitcodeGeneration() { wg.Add(1) go execCompile(compilerExecName, pr, &wg, &ok) wg.Wait() @@ -103,7 +105,7 @@ func Compile(args []string, compiler string) (exitCode int) { // Compiles bitcode files and mutates the list of bc->obj links to perform + the list of // new object files to link -func buildAndAttachBitcode(compilerExecName string, pr parserResult, bcObjLinks *[]bitcodeToObjectLink, newObjectFiles *[]string, wg *sync.WaitGroup) { +func buildAndAttachBitcode(compilerExecName string, pr ParserResult, bcObjLinks *[]bitcodeToObjectLink, newObjectFiles *[]string, wg *sync.WaitGroup) { defer (*wg).Done() var hidden = !pr.IsCompileOnly @@ -207,15 +209,18 @@ func attachBitcodePathToObject(bcFile, objFile string) (success bool) { return } -func compileTimeLinkFiles(compilerExecName string, pr parserResult, objFiles []string) { +func compileTimeLinkFiles(compilerExecName string, pr ParserResult, objFiles []string) { var outputFile = pr.OutputFilename if outputFile == "" { outputFile = "a.out" } - args := objFiles - for _, larg := range pr.LinkArgs { - args = append(args, larg) + args := []string{} + //iam: unclear if this is necessary here + if pr.IsLTO { + args = append(args, LLVMLtoLDFLAGS...) } + args = append(args, objFiles...) + args = append(args, pr.LinkArgs...) args = append(args, "-o", outputFile) success, err := execCmd(compilerExecName, args, "") if !success { @@ -226,7 +231,7 @@ func compileTimeLinkFiles(compilerExecName string, pr parserResult, objFiles []s } // Tries to build the specified source file to object -func buildObjectFile(compilerExecName string, pr parserResult, srcFile string, objFile string) (success bool) { +func buildObjectFile(compilerExecName string, pr ParserResult, srcFile string, objFile string) (success bool) { args := pr.CompileArgs[:] args = append(args, srcFile, "-c", "-o", objFile) success, err := execCmd(compilerExecName, args, "") @@ -239,7 +244,7 @@ func buildObjectFile(compilerExecName string, pr parserResult, srcFile string, o } // Tries to build the specified source file to bitcode -func buildBitcodeFile(compilerExecName string, pr parserResult, srcFile string, bcFile string) (success bool) { +func buildBitcodeFile(compilerExecName string, pr ParserResult, srcFile string, bcFile string) (success bool) { args := pr.CompileArgs[:] //iam: 03/24/2020 extend with the LLVM_BITCODE_GENERATION_FLAGS if any. args = append(args, LLVMbcGen...) @@ -253,14 +258,57 @@ func buildBitcodeFile(compilerExecName string, pr parserResult, srcFile string, return } -// Tries to build object file -func execCompile(compilerExecName string, pr parserResult, wg *sync.WaitGroup, ok *bool) { +// Tries to build object file or link... +func execCompile(compilerExecName string, pr ParserResult, wg *sync.WaitGroup, ok *bool) { defer (*wg).Done() //iam: strickly speaking we should do more work here depending on whether this is // a compile only, a link only, or ... // But for the now, we just remove forbidden arguments var success bool var err error + // start afresh + arguments := []string{} + // we are linking rather than compiling + if len(pr.InputFiles) == 0 && len(pr.LinkArgs) > 0 { + if pr.IsLTO { + arguments = append(arguments, LLVMLtoLDFLAGS...) + } + } + //iam: this is clunky. is there a better way? + if len(pr.ForbiddenFlags) > 0 { + for _, arg := range pr.InputList { + found := false + for _, bad := range pr.ForbiddenFlags { + if bad == arg { + found = true + break + } + } + if !found { + arguments = append(arguments, arg) + } + } + } else { + arguments = append(arguments, pr.InputList...) + } + LogDebug("Calling execCmd(%v, %v)", compilerExecName, arguments) + success, err = execCmd(compilerExecName, arguments, "") + if !success { + LogError("Failed to compile using given arguments:\n%v %v\nexit status: %v\n", compilerExecName, arguments, err) + *ok = false + } +} + +// Tries to build object file + +func oldExecCompile(compilerExecName string, pr ParserResult, wg *sync.WaitGroup, ok *bool) { + defer (*wg).Done() + //iam: strickly speaking we should do more work here depending on whether this is + // a compile only, a link only, or ... + // But for the now, we just remove forbidden arguments + var success bool + var err error + if len(pr.ForbiddenFlags) > 0 { filteredArgs := pr.InputList[:0] for _, arg := range pr.InputList { diff --git a/shared/constants.go b/shared/constants.go index 72c0a9b..10facc4 100644 --- a/shared/constants.go +++ b/shared/constants.go @@ -53,9 +53,11 @@ package shared // See https://github.com/travitch/whole-program-llvm/issues/96 for details. // // 1.2.7 August 4 2020 William Woodruff's (@woodruffw) tweaks to the get-bc command (a strict mode). +// +// 1.2.8 October 11 2020 Wensheng Tang's (@legendtang) requests and fixes to the -flto issues. -const gllvmVersion = "1.2.7" -const gllvmReleaseDate = "August 4 2020" +const gllvmVersion = "1.2.8" +const gllvmReleaseDate = "October 11 2020" const osDARWIN = "darwin" const osLINUX = "linux" diff --git a/shared/environment.go b/shared/environment.go index 999b5f9..f1191fb 100644 --- a/shared/environment.go +++ b/shared/environment.go @@ -86,6 +86,9 @@ var LLVMLd string //LLVMbcGen is the list of args to pass to clang during the bitcode generation step. var LLVMbcGen []string +//LLVMLtoLDFLAGS is the list of extra flags to pass to the linking steps, when unde -flto +var LLVMLtoLDFLAGS []string + const ( envpath = "LLVM_COMPILER_PATH" envcc = "LLVM_CC_NAME" @@ -102,6 +105,9 @@ const ( //iam: 03/24/2020 new feature to pass things like "-flto -fwhole-program-vtables" // to clang during the bitcode generation step envbcgen = "LLVM_BITCODE_GENERATION_FLAGS" + //iam: 10/11/2020 extra linking arguments to add to the linking step when we are doing + // link time optimization. + envltolink = "LTO_LINKING_FLAGS" ) func init() { @@ -138,6 +144,7 @@ func ResetEnvironment() { LLVMObjcopy = "" LLVMLd = "" LLVMbcGen = []string{} + LLVMLtoLDFLAGS = []string{} } // FetchEnvironment is used in initializing our globals, it is also used in testing @@ -158,5 +165,5 @@ func FetchEnvironment() { LLVMLd = os.Getenv(envld) LLVMbcGen = strings.Fields(os.Getenv(envbcgen)) - + LLVMLtoLDFLAGS = strings.Fields(os.Getenv(envltolink)) } diff --git a/shared/parser.go b/shared/parser.go index 0e28c9a..7eda005 100644 --- a/shared/parser.go +++ b/shared/parser.go @@ -43,7 +43,8 @@ import ( "strings" ) -type parserResult struct { +//ParserResult is the result of parsing and partioning the command line arguments. +type ParserResult struct { InputList []string InputFiles []string ObjectFiles []string @@ -58,9 +59,49 @@ type parserResult struct { IsAssembly bool IsCompileOnly bool IsEmitLLVM bool + IsLTO bool IsPrintOnly bool } +const parserResultFormat = ` +InputList: %v +InputFiles: %v +ObjectFiles: %v +OutputFilename: %v +CompileArgs: %v +LinkArgs: %v +ForbiddenFlags: %v +IsVerbose: %v +IsDependencyOnly: %v +IsPreprocessOnly: %v +IsAssembleOnly: %v +IsAssembly: %v +IsCompileOnly: %v +IsEmitLLVM: %v +IsLTO: %v +IsPrintOnly: %v +` + +func (pr *ParserResult) String() string { + return fmt.Sprintf(parserResultFormat, + pr.InputList, + pr.InputFiles, + pr.ObjectFiles, + pr.OutputFilename, + pr.CompileArgs, + pr.LinkArgs, + pr.ForbiddenFlags, + pr.IsVerbose, + pr.IsDependencyOnly, + pr.IsPreprocessOnly, + pr.IsAssembleOnly, + pr.IsAssembly, + pr.IsCompileOnly, + pr.IsEmitLLVM, + pr.IsLTO, + pr.IsPrintOnly) +} + type flagInfo struct { arity int handler func(string, []string) @@ -71,31 +112,50 @@ type argPattern struct { finfo flagInfo } -func skipBitcodeGeneration(pr parserResult) bool { +func (pr *ParserResult) skipBitcodeGeneration() bool { + reason := "No particular reason" + retval := false + squark := LogDebug if LLVMConfigureOnly != "" { - return true + reason = "are in configure only mode" + retval = true + } else if len(pr.InputFiles) == 0 { + reason = "did not see any input files" + retval = true + } else if pr.IsEmitLLVM { + squark = LogWarning + reason = "are in emit-llvm mode, and so the compiler is doing the job for us" + retval = true + } else if pr.IsLTO { + squark = LogWarning + reason = "are doing link time optimization, and so the compiler is doing the job for us" + retval = true + } else if pr.IsAssembly { + reason = "the input file(s) are written in assembly" + squark = LogWarning + retval = true + } else if pr.IsAssembleOnly { + reason = "are assembling only, and so have nowhere to embed the path of the bitcode" + retval = true + } else if pr.IsDependencyOnly && !pr.IsCompileOnly { + reason = "are only computing dependencies at this stage" + retval = true + } else if pr.IsPreprocessOnly { + reason = "are in preprocess only mode" + retval = true + } else if pr.IsPrintOnly { + reason = "are in print only mode, and so have nowhere to embed the path of the bitcode" + retval = true } - if len(pr.InputFiles) == 0 || - pr.IsEmitLLVM || - pr.IsAssembly || - pr.IsAssembleOnly || - (pr.IsDependencyOnly && !pr.IsCompileOnly) || - pr.IsPreprocessOnly || - pr.IsPrintOnly { - // Lots of reasons why we don't produce bitcode. But IsEmitLLVM is the most unusual. - if pr.IsEmitLLVM { - // Either we are being called with -emit-llvm or -flto. Either way all bets are off. - LogWarning("The compiler is producing bitcode instead of object code. get-bc is unlikely to work: %v\n", pr.OutputFilename) - } - - return true + if retval { + squark(" We are skipping bitcode generation because we %v.\n", reason) } - return false - + return retval } -func parse(argList []string) parserResult { - var pr = parserResult{} +//Parse analyzes the command line aruguments and returns the result of that analysis. +func Parse(argList []string) ParserResult { + var pr = ParserResult{} pr.InputList = argList var argsExactMatches = map[string]flagInfo{ @@ -350,7 +410,7 @@ func parse(argList []string) parserResult { } // Return the object and bc filenames that correspond to the i-th source file -func getArtifactNames(pr parserResult, srcFileIndex int, hidden bool) (objBase string, bcBase string) { +func getArtifactNames(pr ParserResult, srcFileIndex int, hidden bool) (objBase string, bcBase string) { if len(pr.InputFiles) == 1 && pr.IsCompileOnly && len(pr.OutputFilename) > 0 { objBase = pr.OutputFilename dir, baseName := path.Split(objBase) @@ -381,7 +441,7 @@ func getHashedPath(path string) string { return hash } -func (pr *parserResult) inputFileCallback(flag string, _ []string) { +func (pr *ParserResult) inputFileCallback(flag string, _ []string) { var regExp = regexp.MustCompile(`\.(s|S)$`) pr.InputFiles = append(pr.InputFiles, flag) if regExp.MatchString(flag) { @@ -389,11 +449,11 @@ func (pr *parserResult) inputFileCallback(flag string, _ []string) { } } -func (pr *parserResult) outputFileCallback(_ string, args []string) { +func (pr *ParserResult) outputFileCallback(_ string, args []string) { pr.OutputFilename = args[0] } -func (pr *parserResult) objectFileCallback(flag string, _ []string) { +func (pr *ParserResult) objectFileCallback(flag string, _ []string) { // FIXME: the object file is appended to ObjectFiles that // is used nowhere else in the code pr.ObjectFiles = append(pr.ObjectFiles, flag) @@ -402,76 +462,76 @@ func (pr *parserResult) objectFileCallback(flag string, _ []string) { pr.LinkArgs = append(pr.LinkArgs, flag) } -func (pr *parserResult) preprocessOnlyCallback(_ string, _ []string) { +func (pr *ParserResult) preprocessOnlyCallback(_ string, _ []string) { pr.IsPreprocessOnly = true } -func (pr *parserResult) dependencyOnlyCallback(flag string, _ []string) { +func (pr *ParserResult) dependencyOnlyCallback(flag string, _ []string) { pr.IsDependencyOnly = true pr.CompileArgs = append(pr.CompileArgs, flag) } -func (pr *parserResult) printOnlyCallback(flag string, _ []string) { +func (pr *ParserResult) printOnlyCallback(flag string, _ []string) { pr.IsPrintOnly = true } -func (pr *parserResult) assembleOnlyCallback(_ string, _ []string) { +func (pr *ParserResult) assembleOnlyCallback(_ string, _ []string) { pr.IsAssembleOnly = true } -func (pr *parserResult) verboseFlagCallback(_ string, _ []string) { +func (pr *ParserResult) verboseFlagCallback(_ string, _ []string) { pr.IsVerbose = true } -func (pr *parserResult) compileOnlyCallback(_ string, _ []string) { +func (pr *ParserResult) compileOnlyCallback(_ string, _ []string) { pr.IsCompileOnly = true } -func (pr *parserResult) emitLLVMCallback(_ string, _ []string) { +func (pr *ParserResult) emitLLVMCallback(_ string, _ []string) { pr.IsCompileOnly = true pr.IsEmitLLVM = true } -func (pr *parserResult) linkTimeOptimizationCallback(_ string, _ []string) { - pr.IsEmitLLVM = true +func (pr *ParserResult) linkTimeOptimizationCallback(_ string, _ []string) { + pr.IsLTO = true } -func (pr *parserResult) linkUnaryCallback(flag string, _ []string) { +func (pr *ParserResult) linkUnaryCallback(flag string, _ []string) { pr.LinkArgs = append(pr.LinkArgs, flag) } -func (pr *parserResult) compileUnaryCallback(flag string, _ []string) { +func (pr *ParserResult) compileUnaryCallback(flag string, _ []string) { pr.CompileArgs = append(pr.CompileArgs, flag) } -func (pr *parserResult) warningLinkUnaryCallback(flag string, _ []string) { +func (pr *ParserResult) warningLinkUnaryCallback(flag string, _ []string) { LogWarning("The flag %v cannot be used with this tool, we ignore it, else we lose the bitcode section.\n", flag) pr.ForbiddenFlags = append(pr.ForbiddenFlags, flag) } -func (pr *parserResult) defaultBinaryCallback(_ string, _ []string) { +func (pr *ParserResult) defaultBinaryCallback(_ string, _ []string) { // Do nothing } -func (pr *parserResult) dependencyBinaryCallback(flag string, args []string) { +func (pr *ParserResult) dependencyBinaryCallback(flag string, args []string) { pr.CompileArgs = append(pr.CompileArgs, flag, args[0]) pr.IsDependencyOnly = true } -func (pr *parserResult) compileBinaryCallback(flag string, args []string) { +func (pr *ParserResult) compileBinaryCallback(flag string, args []string) { pr.CompileArgs = append(pr.CompileArgs, flag, args[0]) } -func (pr *parserResult) linkBinaryCallback(flag string, args []string) { +func (pr *ParserResult) linkBinaryCallback(flag string, args []string) { pr.LinkArgs = append(pr.LinkArgs, flag, args[0]) } -func (pr *parserResult) compileLinkUnaryCallback(flag string, _ []string) { +func (pr *ParserResult) compileLinkUnaryCallback(flag string, _ []string) { pr.LinkArgs = append(pr.LinkArgs, flag) pr.CompileArgs = append(pr.CompileArgs, flag) } -func (pr *parserResult) compileLinkBinaryCallback(flag string, args []string) { +func (pr *ParserResult) compileLinkBinaryCallback(flag string, args []string) { pr.LinkArgs = append(pr.LinkArgs, flag, args[0]) pr.CompileArgs = append(pr.CompileArgs, flag, args[0]) }