## Building the linux kernel from llvm bitcode requires writing scripts to copy, extract and recompile a lot of files in the right order. ## A script is only valid for one configuration of the kernel ## The goal of this Python script is to automate the script building process for any configuration of the kernel. ## It takes one or more argument : the first argument is where to write the final script, and all the others are paths to files of which we should not use the bitcode. import sys,os import subprocess # This is where most of the script gets written. # This function takes as argument: # a list of path that should not be translated into bitcode # the depth from the root (this is a recursive function) # the path from the root to the current folder (its length should be depth) def write_script(excluded_paths, depth, base_dir): builtin = open (base_dir+"arbi","w") subprocess.call(["ar", "-t", base_dir+"built-in.o"],stdout=builtin ) builtin.close() builtin = open(base_dir+"arbi","r") out.writelines("mkdir -p $build_home/built-ins/"+base_dir+" \n") # In directories we store the list of depth 1 directories (and files) found in builtin directories=[] for line in builtin.readlines(): line=line.split('\n')[0] words=line.split('/') if words[0]=="arch": # We had to make an exception for the arch folder which does not have a built-in.o at the root. words[0]=words[0]+'/'+words[1] if words[2] in archbi: #A second exception for arch, not all files in the folder are referenced in the arch/x86/built-in.o. We store the exceptions in the archi list words[0]=words[0]+'/'+words[2] if words[depth] not in directories and line not in standalone_objects: #Some objects are not listed in a built-in.o except at the root, so we include them separately in order not to mess the order of linking directories.append(words[depth]) # folders in which there are excluded folders which we will need to act on recursively excluded_roots = [path[depth] for path in excluded_paths] for direc in directories: print base_dir+direc # the folder contains an excluded folder if direc in excluded_roots: # Check if we have an excluded folder if base_dir+direc in excluded_dirs: out.writelines("convert-thin-archive.sh "+base_dir+direc+"/built-in.o \n") out.writelines("cp "+ base_dir +direc+"/built-in.o.new $build_home/built-ins/"+base_dir+direc+"bi.o \n \n") link_args.writelines("built-ins/"+base_dir + direc +"bi.o ") # Else we filter the excluded_path list for only relevant stuff and we call the recursion on that folder else: relevant_excluded = [path for path in excluded_paths if (path[depth]==direc and len(path)>depth+1) ] if relevant_excluded: write_script(relevant_excluded,depth+1,base_dir+direc+"/") else: #If the "directory" is a file, we copy it into the relevant objects folder and compile it if direc[-2:] == ".o": out.writelines("get-bc -b "+base_dir+direc+"\n") out.writelines("mkdir -p $build_home/built-ins/"+base_dir+"objects \n") out.writelines("cp "+ base_dir+direc+".bc $build_home/built-ins/"+base_dir+"objects \n") out.writelines("clang -c -no-integrated-as -mcmodel=kernel -o $build_home/built-ins/"+base_dir+ direc + " $build_home/built-ins/"+base_dir+"objects/" + direc+".bc \n \n") # We then add it to the linker arguments file link_args.writelines("built-ins/"+base_dir + direc +" ") # When dealing with a folder, we get the bitcode from the built-in.o file and check for errors in the log. # For each file that does not have a bitcode version (compiled straight from assembly) we copy it into the build folder directly and add it to the linker args else: if os.path.isfile("/vagrant/wrapper-logs/wrapper.log"): direc_no_slash=direc.split('/')[0] os.rename("/vagrant/wrapper-logs/wrapper.log","/vagrant/wrapper-logs/before_"+direc_no_slash+".log") #os.remove("/vagrant/wrapper-logs/wrapper.log") path = base_dir + direc +"/built-in.o" subprocess.call(["get-bc", "-b", path ]) subprocess.call(["touch","/vagrant/wrapper-logs/wrapper.log"]) # Checking to see if any file failed to be extracted to bitcode llvm_log=open("/vagrant/wrapper-logs/wrapper.log","r") assembly_objects=[] for line in llvm_log.readlines(): if len(line)>=54 and line[:54]=="WARNING:Error reading the .llvm_bc section of ELF file": assembly_objects.append(line[55:-2]) llvm_log.close() # Deal with those files if assembly_objects: out.writelines("mkdir -p $build_home/built-ins/" + base_dir +direc+"\n") for asf in assembly_objects: out.writelines("cp "+ asf + " $build_home/built-ins/" + base_dir +direc +"\n") filename= asf.split('/')[-1] link_args.writelines("built-ins/"+base_dir + direc +'/'+filename+" ") # Deal with the rest if os.path.isfile(base_dir + direc +"/built-in.o.a.bc"): out.writelines("cp "+ base_dir + direc +"/built-in.o.a.bc $build_home/built-ins/"+ base_dir + direc+"bi.o.bc \n") out.writelines("clang -c -no-integrated-as -mcmodel=kernel -o $build_home/built-ins/"+ base_dir + direc + "bibc.o $build_home/built-ins/" + base_dir+direc+"bi.o.bc \n \n") link_args.writelines("built-ins/"+base_dir + direc +"bibc.o ") builtin.close() # excluded_dirs is a list where the folders excluded from bitcode translation will be stored excluded_dirs=[] if len(sys.argv) > 2: excluded_dirs=sys.argv[2:] # Output file path script_out= sys.argv[1] #We transform excluded_dirs into a 2D array for better access to individual folders in the path excluded=[path.split('/') for path in excluded_dirs] out = open(script_out,"w+") link_args = open("/home/vagrant/standalone-build/link-args","w+") out.writelines("# Script written by the built-in-parsing.py script \n") out.writelines("export build_home=/home/vagrant/standalone-build\n") # List exceptions standalone_objects = ["arch/x86/kernel/head_64.o","arch/x86/kernel/head64.o","arch/x86/kernel/ebda.o","arch/x86/kernel/platform-quirks.o"]#,"usr/initramfs_data.o"] archbi=["lib","pci","video","power"] # Calling the main function write_script(excluded,0,"") # Dealing with both lib files out.writelines("get-bc -b lib/lib.a \n") out.writelines("mkdir $build_home/lib") out.writelines("cp lib/lib.a.bc $build_home/lib \n") out.writelines("clang -c -no-integrated-as -mcmodel=kernel -o $build_home/lib/lib.a.o $build_home/lib/lib.a.bc \n") if os.path.isfile("/vagrant/wrapper-logs/wrapper.log"): os.rename("/vagrant/wrapper-logs/wrapper.log","/vagrant/wrapper-logs/before_lib.log") #os.remove("/vagrant/wrapper-logs/wrapper.log") subprocess.call(["get-bc", "-b", "arch/x86/lib/lib.a" ]) subprocess.call(["touch","/vagrant/wrapper-logs/wrapper.log"]) llvm_log=open("/vagrant/wrapper-logs/wrapper.log","r") assembly_objects=[] for line in llvm_log.readlines(): if len(line)>=54 and line[:54]=="WARNING:Error reading the .llvm_bc section of ELF file": assembly_objects.append(line[55:-2]) if assembly_objects: out.writelines("mkdir -p $build_home/arch/x86/lib/objects \n") for asf in assembly_objects: out.writelines("cp "+ asf + " $build_home/arch/x86/lib/objects/ \n") filename= asf.split('/')[-1] #link_args.writelines("arch/x86/lib/objects/" +filename+" ") ##commented to keep the order of linked files out.writelines("cp arch/x86/lib/lib.a.bc $build_home/arch/x86/lib/lib.a.bc \n") out.writelines("clang -c -no-integrated-as -mcmodel=kernel -o $build_home/arch/x86/lib/lib.a.o $build_home/arch/x86/lib/lib.a.bc \n \n") #Deal with individual files out.writelines("cp arch/x86/kernel/vmlinux.lds $build_home \n") out.writelines("cp .tmp_kallsyms2.o $build_home \n") for sto in standalone_objects: out.writelines("cp --parents "+ sto+" $build_home \n") out.writelines("\n#linking command \n") out.writelines("cd $build_home \n") # Final linking command out.writelines("ld --build-id -T vmlinux.lds --whole-archive ") for sto in standalone_objects: out.writelines(sto+" ") out.writelines("\@link-args ") out.writelines("--no-whole-archive --start-group lib/lib.a.o arch/x86/lib/lib.a.o arch/x86/lib/objects/* .tmp_kallsyms2.o --end-group -o vmlinux")