#!/usr/bin/perl -w
#$Id: objcache 46153 2007-10-19 00:26:07Z wsnyder $
######################################################################
#
# This program is Copyright 2002-2007 by Wilson Snyder.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of either the GNU General Public License or the
# Perl Artistic License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#                                                                           
######################################################################

require 5.006_001;

# For testing
use FindBin qw($RealBin);
use lib "$RealBin/blib/lib";  # For local testing
use lib "$RealBin/blib/arch"; # For local testing

use Getopt::Long;
use IO::File;
use IO::Pipe;
use Pod::Usage;
use File::Copy;
use POSIX qw(:sys_wait_h);

use Make::Cache::Gcc;
use strict;

our $Debug;

our $VERSION = '1.043';

#======================================================================

# Directory where cache is stored
our $Cache_Dir = ($ENV{OBJCACHE_DIR}||"/usr/local/common/lib/objcache");

# Allow another calling program to override flag defaults
use vars qw(@ObjCache_Additional_Flags);

#======================================================================
# main

autoflush STDOUT 1;
autoflush STDERR 1;
umask 0;
srand;

my @params = ();

our $Opt_Loud = 1;
our $Opt_Read = 0;
my $opt_clean;
our $opt_distcc;
my $opt_dump;
our $opt_dump_rm;
our $opt_icecream;
our $opt_jobs;
our $opt_link;
our $opt_gcc;
our $opt_wrapper;
our $Opt_Write = 0;
our @Opt_Okdir;

# Before reading standards, collect help & debug flags
Getopt::Long::config ("pass_through", "no_auto_abbrev");
if (!GetOptions (
		 "help"		=> \&usage,
		 "debug"	=> \&debug,
		 "quiet!"	=> sub {shift; $Opt_Loud = !shift;},
		 "clean!"	=> \$opt_clean,
		 "distcc!"	=> \$opt_distcc,
		 "dump!"	=> \$opt_dump,
		 "dumprm!"	=> \$opt_dump_rm,
		 "gcc!"		=> \$opt_gcc,
		 "icecream!"	=> \$opt_icecream,
		 "link!"	=> \$opt_link,
		 "jobs:s"	=> \$opt_jobs,
		 "okdir=s"	=> sub {shift; push @Opt_Okdir, shift;},
		 "read!"	=> \$Opt_Read,
		 "wrapper!"	=> \$opt_wrapper,
		 "write!"	=> \$Opt_Write,
		 # Program and it's parameters
		 "<>"		=> \&parameter,
		 )) {
    die "%Error: Bad usage, try 'objcache --help'\n";
}
# Read rest of command line
push @params, @ARGV;
print "OBJCACHE R$Opt_Read W$Opt_Write: ",join(',',@params),"\n" if $Debug;

$opt_dump = 1 if $opt_dump_rm;

if (defined $opt_jobs) {
    my @hosts = (split(/[ :]/,$opt_jobs||$ENV{OBJCACHE_HOSTS}||""));
    my $jobs = $ENV{OBJCACHE_JOBS} || "0";
    $jobs =~ s/^-j//;   # so OBJCACHE_JOBS can have the make '-j #' flag in it
    $jobs ||= ($#hosts + 1);
    $jobs = 1 if $jobs==0;
    $ENV{OBJCACHE_JOBS} = $jobs;
    print "$jobs\n";
    exit(0);
}
if ($opt_clean) {
    my $oc = Make::Cache::Gcc->new (dir=>$Cache_Dir,);
    $oc->clean();
    !$params[0] or die "objcache: %Error: --clean doesn't take any other arguments\n";
    exit(0);
}
if ($opt_dump) {
    my $oc = Make::Cache::Gcc->new (dir=>$Cache_Dir,);
    $oc->dump(rm=>$opt_dump_rm);
    !$params[0] or die "objcache: %Error: --dump doesn't take any other arguments\n";
    exit(0);
}

my @ocparam = (dir=>$Cache_Dir,
	       remote_hosts=>[split(/[ :]/,$ENV{OBJCACHE_HOSTS}||"")],
	       min_remote_runtime => 3,  # Secs
	       min_cache_runtime  => 2,  # Secs
	       nfs_wait => ($ENV{OBJCACHE_NFS_WAIT} || 4),
	       edit_line_refs => {},
	       ok_include_regexps => [],
	       distcc => $opt_distcc,
	       icecream => $opt_icecream,
	       force_gcc => $opt_gcc,
	       link => $opt_link,
	       wrapper => $opt_wrapper,
	       file_env => [ # Hash of global mnemonic and local value for global_filename funcs
			     PWD => Cwd::getcwd(),
			     PWD => ($ENV{PWD}||Cwd::getcwd()),
			     ],
	       @ObjCache_Additional_Flags,	# Allow another calling program to override these defaults
	       );

# Make class with the compile time parameters
my $ochunt = Make::Cache::Gcc->new (@ocparam);
$ochunt->ok_include_regexps(qr!^/usr/include/!);
$ochunt->ok_include_regexps(qr!^/usr/lib/!);
$ochunt->ok_include_regexps(qr!^/usr/lib64/!);
$ochunt->ok_include_regexps(qr!^/usr/local/include/!);
$ochunt->ok_include_regexps(qr!^/usr/share/!);
foreach (@Opt_Okdir) { $ochunt->ok_include_regexps("^".quotemeta($_)); }

$ochunt->cmds_lcl(@params);
$ochunt->parse_cmds;
$ochunt->tgts_unlink;   # Unlink all generated files, so there's no way a objcache bug could cache them

my @tgts = $ochunt->tgts_lcl;
(my $srcfile = $tgts[0]) =~ s/.*\///;   # Make it small so prints look nice

$ochunt->preproc;
my $ochit = $ochunt->find_hit if $Opt_Read;

if ($ochit) {
    print "      Compiling $srcfile...  Object Cache Hit\n" if $Opt_Loud;
    $ochit->restore;
    exit(0);
} else {
    if ($Opt_Loud) {
	# Print all messages in single print statement, else make -j may split the lines.
	my $msg = "      Compiling $srcfile...";
	$msg .= "  Est ".Make::Cache::Runtime::format_time($ochunt->runtime) if defined $ochunt->runtime;
	$msg .= " (on ".$ochunt->host.")" if $ochunt->host;
	$msg .= "\n";
	print $msg;
    }
}

# Run command passed
$ochunt->execute;

# Fast compile, don't spend time in the cache, just update the runtime and build each time
#if ($ochunt->runtime < $Remote_Runtime) {
#	This was determined to be a bad idea, due to fluctuations around the critical time

if ($Opt_Write) {
    $ochunt->encache;
} else {
    $ochunt->runtime_write;
}
#print "      Compiling $srcfile took ".Make::Cache::Runtime::format_time($ochunt->runtime)."\n" if $Opt_Loud;

if ($Debug && $Opt_Read && $Opt_Write) {
    $ochunt->clear_hash_cache;
    my $mc = $ochunt->new();
    my $hit = $mc->find_hit();
    if (!$hit) {
	warn "%Warning: Didn't hit own written hash!\n";
    }
}

#----------------------------------------------------------------------

sub usage {
    print '$Id: objcache 46153 2007-10-19 00:26:07Z wsnyder $ ', "\n";
    pod2usage(-verbose=>2, -exitval => 2);
    exit (1);
}

sub debug {
    $Debug = 1;
    $Make::Cache::Gcc::Debug = 1;
}

sub parameter {
    my $param = shift;
    if ($param =~ /^-/) {
	die "objcache: %Error: Unknown switch before program: $param\n";
    }
    else {
	push @params, $param;
	die ("!FINISH");	# Magic to tell Getopt::Long to ignore other switches
    }
}

1;
#######################################################################
__END__

=pod

=head1 NAME

objcache - Cache results of running gcc/ghs on argument list

=head1 SYNOPSIS

  objcache --read --write g++ <normal_gcc/ghs_command_line>

=head1 DESCRIPTION

objcache is called with a full g++ or cxppc command line.  It acts as if
the compiler is called directly with all arguments.

With --read and --write, objcache returns almost instantly when the same
source is recompiled.  It does this by caching a hash of the preprocessed
gcc source files.  If gcc is invoked with the same inputs, the cache
returns the object files without needing to invoke the compiler.

=head1 DETAILS

GCC is run in preprocessor mode to create a single source file.  This
source file is then hashed.  Likewise any compiler switches are hashed, but
with any define related switches (-Dfoo -Dfoo=value -Ufoo) stripped out as
they are represented in the preprocessor output.  (This increases cache
hits when there are many #ifdef controlled compiles going on.)

The source hash is then looked up in the cache.  If it hits, the objects
are copied from the cache into the local directory, and objcache exits.
The files on disk will thus look like the compile finished, but much faster.

If the source hash misses, the compiler is invoked.  The output of the
compiler is written to the cache.  objcache also determines how long the
compile took (for informing the user), and may run the compile on another
machine.

=head1 EXAMPLE MAKEFILE

This example will use the cache, and compile on all machines in the network
with the "gcc" class.  It's also written to work if the objcache is not
installed.  This uses the Schedule::Load package to determine what machines
have free resources in the network.

  ifeq ($(SLCHOOSED_HOST),)
   export OBJCACHE := 
  else
   export OBJCACHE_HOSTS := $(shell rschedule --class class_gcc hostnames)
   export OBJCACHE_JOBS := -j $(shell objcache --jobs "$(OBJCACHE_HOSTS)")
   export OBJCACHE := @objcache --read --write
  endif

  %.o:	%.cpp
	$(OBJCACHE) ${CXX} ${CPPFLAGS} -c $<

If you are using a submake (where one makefile calls another), you may
place the export lines in the top level makefile.  Then, spawn the submakes
using the number of jobs calculated by objcache --jobs:

  top_level_target:
        $(MAKE) $(OBJCACHE_JOBS) top_level_target


=head1 ARGUMENTS

=over 4

=item --help

Displays this message and program version and exits.

=item --clean

Remove any files older then one day from the cache.

=item --distcc

Use the distcc program to distribute compile jobs, rather then logging in
remotely and executing the compilation command.

=item --gcc

Ignore the compiler name specified on the command line, and assume the
compiler has the same command line options as GCC.

=item --dump

For debugging, show the state of the cache.

=item --dumprm

Show the state of the cache, with the command line command needed to flush
that entry.

=item --icecream

Use the icecream program to distribute compile jobs, rather then logging in
remotely and executing the compilation command.

=item --jobs

Return a suggestion for the number of parallel make jobs to be run.  This
is the contents of the OBJCACHE_JOBS variable, or if not set, a count of
the number of hosts listed in the OBJCACHE_HOSTS variable.  With a
argument, use the argument instead of the OBJCACHE_HOSTS variable.

=item --link

When a object file hits, create a symlink to the master version, rather
then copying.  This may result in better performance when there is a single
user, though worse performance when builds are local and the repository on
a global disk.

=item --okdir

Specify a directory that should avoid the strange directory warning.  Use
this with caution, as absolute paths may greatly decrease hit rates between
different users.

=item --read

Read the cache and use cached objects if they exist.

=item --noruntime

Disable caching the execution time of the compile, nor show the runtime
when compling.

=item --wrapper

Interpret the first program on the command line as a wrapper script, and
get the real compiler name from the next non-switch argument.  Useful for
applications like purify.

=item --write

Write the cache with compiled objects.

=back

=head1 ENVIRONMENT

=over 4

=item OBJCACHE_DIR

Specifies the directory containing the cache.  Defaults to
/usr/local/common/lib/objcache.  Under this is a directory based on a hash
of the target name.  Under that is a directory based on a hash of the
source file and compile switches, then finally .digest and .t# directory
entries for each hash and target file.  You can remove all of the files
in this directory to empty the cache.

=item OBJCACHE_RUNTIME_DIR

Specifies the directory containing the runtime database.  Defaults to
/usr/local/common/lib/runtime.

=item OBJCACHE_HOSTS

Specifies a comma separated list of hosts to run compiles on.  When a
compile needs to be run, objcache will pick a random host from this list,
then remote shell to run the compile.  This allows a "make -j" run to use
many machines in parallel.  Defaults to not remote shell.

=item OBJCACHE_JOBS

Specifies the forced return value for objcache --jobs.  If there is a
leading -j, it will be stripped.

=item OBJCACHE_NFS_WAIT

Specifies the number of seconds to wait for a generated file written on one
machine to become visible on another machine, before signalling an error.
Defaults to 4 seconds, but may need to be increased on slow networks.

=item OBJCACHE_RSH

Specifies the program name to use for remote shells.  Defaults to rsh.

=back

=head1 DISTRIBUTION

The latest version is available from CPAN and from L<http://www.veripool.com/>.

Copyright 2000-2007 by Wilson Snyder.  This package is free software; you
can redistribute it and/or modify it under the terms of either the GNU
Lesser General Public License or the Perl Artistic License.

=head1 AUTHORS

Wilson Snyder <wsnyder@wsnyder.org>

=head1 SEE ALSO

L<Make::Cache>

=cut

######################################################################
### Local Variables:
### compile-command: "./objcache "
### End: