Tjs Useful Scripts

sort_ads.pl

#!/usr/bin/env perl
use strict;
use warnings;

my %ad;

for my $line ( <STDIN> ) {
  if ($line =~ /^\s*$/) {
    # print out the previous add sorted by key then reset the hash for a new ad.
    if (%ad) {
       for (sort keys %ad) { print $_; }
       undef %ad;
    }
  }
  $ad{$line} = 1;
}

# print the last ad sorted by key
if (%ad) {
   for (sort keys %ad) { print $_; }
   undef %ad;
}

cmakeit.bat

@echo off
setlocal

set _noregen=-DCMAKE_SUPPRESS_REGENERATION:BOOL=TRUE
if "%1"=="regen" (
   shift
   set _noregen=-DCMAKE_SUPPRESS_REGENERATION:BOOL=FALSE
)

if "%1"=="analyze" (
   shift
   set _analyze=-DMSVC_ANALYZE:BOOL=TRUE
)
if "%1"=="noanalyze" (
   shift
   set _analyze=-DMSVC_ANALYZE:BOOL=FALSE
)

if "%1"=="tests" (
   shift
   set _test=-DBUILD_TESTING:BOOL=TRUE -DWITH_BOOST:BOOL=TRUE
)
if "%1"=="buildid" (
   set _buildid=-DBUILDID:STRING="%~2"
   shift
   shift
)

if exist .\.git\config goto :nothere
if exist .gitignore goto :nothere

if exist .\src\condor_utils\condor_api.h (
  set _src=
) else (
  set _src=%GIT_HOME%
  choice /M "no sources in current dir, use %GIT_HOME%"
  if errorlevel 2 goto :EOF
)

set _gen="Visual Studio 11"
if "%VS_DIR%"=="C:\vs90" set _gen="Visual Studio 9 2008"
REM set _gen="NMake Makefiles"
if NOT ~%1==~ set _gen=%1

@echo on
cmake -LA CMakeLists.txt -G %_gen% %_noregen% %_analyze% %_test% %_buildid% -D_VERBOSE:BOOL=TRUE %_src%
@goto :EOF

:nothere
@echo You don't want to run this when the current directory is a git repository.

rebuild.bat

@echo off
setlocal

for %%I in (wget.exe) do if "%%~f$PATH:I"=="" (
  echo WGET is not in the path - aborting
  exit /b 1
)

if not exist condor.sln (
  echo There is no Condor.sln file in the current directory
  echo did you forget to run cmakeit.bat?
  exit /b 1
)

set _scriptdir_=%~sdp0
set _counterrs_=call %~sdp0counterrs.bat

:top
if "~%1"=="~" goto doit
goto %1

:debug
set _bldtype_=Debug
goto next

:con
:console
set _console_=true
goto next

:build
set _rebuild_=Build
goto next

:relwithdebinfo
:retail
set _bldtype_=RelWithDebInfo
goto next

:package
set _target_=PACKAGE
goto next

:externals
set _target_=ALL_EXTERN
goto next

:install
set _target_=INSTALL
goto next

:release
set _release_=call %_scriptdir_%release.bat
if "%2"=="test" (
    set _release_=%_release_% test
    shift
)
if "%_bldtype_%"=="Debug" (
    set _release_=%_release_% debug
)
goto next

:next
shift
goto top

:doit
if ~%_rebuild_%==~ set _rebuild_=Rebuild
if ~%_bldtype_%==~ set _bldtype_=RelWithDebInfo
if ~%_target_%==~ set _target_=ALL_BUILD
set _counterrs_=call %_scriptdir_%counterrs.bat %_rebuild_%.out
::@echo '%_counterrs_%'

@if NOT "%_console_%"=="true" goto :build_to_log
@echo on
devenv condor.sln /%_rebuild_% %_bldtype_% /project %_target_%
@echo off
goto :EOF

:build_to_log
@echo on
devenv condor.sln /%_rebuild_% %_bldtype_% /project %_target_% > %_rebuild_%.out
@echo off
%_counterrs_%
%_release_%

counterrs.bat

@echo off
@setlocal
set infile=%1
if "%infile%"=="" set infile=Rebuild.out
set tmpfile=%infile:.out=%_nopre.tmp

:: extract a sorted list of error/warning messages without the "nn>" prefix on the lines.
grep -E "C[0-9]{4}:" %infile% | awk "{gsub(/^[0-9]+>/,\"\")};1" | sort > %tmpfile%

:: set variables to conain counts of various things.
for /F %%I in ('grep -E " [C|E][0-3][0-9]{3}" %infile% ^| sort ^| uniq ^| awk "END{print NR}"') do set errs=%%I
for /F %%I in ('grep -E " C4[0-9]{3}" %infile% ^| sort ^| uniq ^| awk "END{print NR}"') do set warns=%%I
for /F %%I in ('grep -E " C6[0-9]{3}" %infile% ^| sort ^| uniq ^| awk "END{print NR}"') do set anals=%%I
for /F %%I in ('grep -E " C6[0-9]{3}" %infile% ^| grep -I -E "\.h" ^| sort ^| uniq ^| awk "END{print NR}"') do set analhs=%%I

set analmsg=
if NOT "%anals%"=="0" set analmsg=%anals% Analysis warnings
set analhmsg=
if NOT "%analhs%"=="0" set analhmsg=^(%analhs% in headers)
@echo %infile% Raw  : %errs% Errors, %warns% Warnings %analmsg% %analhmsg%

for /F %%I in ('grep -E " [C|E][0-3][0-9]{3}" %tmpfile% ^| sort ^| uniq ^| awk "END{print NR}"') do set errs=%%I
for /F %%I in ('grep -E " C4[0-9]{3}" %tmpfile% ^| sort ^| uniq ^| awk "END{print NR}"') do set warns=%%I
for /F %%I in ('grep -E " C6[0-9]{3}" %tmpfile% ^| sort ^| uniq ^| awk "END{print NR}"') do set anals=%%I
for /F %%I in ('grep -E " C6[0-9]{3}" %tmpfile% ^| grep -I -E "\.h" ^| sort ^| uniq ^| awk "END{print NR}"') do set analhs=%%I

set analmsg=
if NOT "%anals%"=="0" set analmsg=%anals% Analysis warnings
set analhmsg=
if NOT "%analhs%"=="0" set analhmsg=^(%analhs% in headers)
@echo %infile% Uniq : %errs% Errors, %warns% Warnings %analmsg% %analhmsg%

if not "%errs%"=="0" @grep -E "[0-9]*[1-9]+ error\(s\), [0-9]+ warning\(s\)" %infile%

release.bat

@echo off
setlocal

for %%I in (cmake.exe) do if "%%~f$PATH:I"=="" (
  echo cmake is not in the path - aborting
  exit /b 1
)

if not exist cmake_install.cmake (
  echo There is no cmake_install.cmake file in the current directory
  echo did you forget to run cmakeit.bat?
  exit /b 1
)

set _bldtype_=RelWithDebInfo

if "%1"=="msi" goto msi
if "~%1"=="~" goto doit
if "~%1"=="~." set _install_=-DCMAKE_INSTALL_PREFIX=Release
if NOT "~%1"=="~." set _install_=-DCMAKE_INSTALL_PREFIX=%~sf1

:top
if "~%2"=="~" goto doit
goto %2

:debug
set _bldtype_=Debug
goto next

:relwithdebinfo
:retail
set _bldtype_=RelWithDebInfo
goto next

:msi
set _msi_=condor-pre.msi
goto next

:next
shift
goto top

:doit
if ~%_bldtype_%==~ for /F %%I in ('dir /b all_build.dir') do set _bldtype_=%%I

@echo on
cmake.exe -DBUILD_TYPE=%_bldtype_% %_install_% -P cmake_install.cmake > release.out
@echo off
::@findstr /I /C:"condor_master.exe" release.out
for /F %%I in ('grep -c Installing release.out') do set /A install_count=%%I - 1
for /F %%I in ('grep -c Up-to-date release.out') do set /A uptodate_count=%%I + %install_count%
echo Updated %install_count% of %uptodate_count%

:makemsi
if "%_msi_%"=="" goto :EOF
set _msiname_=
for /f "delims== tokens=3" %%I in ('findstr "condor-" msconfig\wix\xml\condor.xsl') do @set _msiname_=%%~I
if NOT "%_msiname_%"=="" set _msi_=%_msiname_%.msi
:: @echo msi is: "%_msi_%"
if not "%_msi_:~0,6%"=="condor" goto :EOF

:: figure out the release path from the release.out log
for /f "tokens=3" %%I in ('grep condor_master.exe release.out') do @set _master_=%%~dpI
set _reldir_=%_master_:\bin\=%
if not EXIST %_reldir_%\bin\condor_master.exe goto :EOF

@echo makeing msi: %_msi_%
@echo on
%_reldir_%\etc\wix\do_wix %_reldir_% %CD%\%_msi_%

prep_for_tests.bat

@setlocal
@echo off
if "%GIT_HOME%"=="" goto :needhome

set testdir=%1
if "%testdir%"=="" set testdir=test%random%
set bldtype=Debug
if "%2"=="retail" set bldtype=RelWithDebInfo

call :doit "%testdir%" %bldtype%
goto :EOF

:needhome
@echo this script requires the GIT_HOME environment variable set to the location of your condor sources
goto :EOF

:usage
@echo Usage: %~n0 [^<testdir^>] [debug | retail]
@echo    creates ^<testdir^> installs condor into it, then builds tests and copies scripts
@echo    and tests into this directory as well.  it crates setenv.bat in this directory
@echo    and starts a shell with it so that you can run tests.
@echo    This script assumes that the current directory is your condor build directory,
@echo    and that the environment variable GIT_HOME points to the htcondor sources.
goto :EOF

:doit
@echo Init tests in %testdir% (%~f1) for %bldtype%

if exist "%testdir%\bin\condor_version.exe" goto :gotbins
mkdir "%testdir%"
call %~sdp0release %testdir% %bldtype%
pushd "%testdir%"
cscript "%GIT_HOME%\msconfig\init_config.wsf" ETC=etc PERSONAL=Y >init_config.log
if not exist log\MasterLog mkdir log
if not exist spool\job_queue.log mkdir spool && mkdir execute
popd

:gotbins
xcopy /Q "%GIT_HOME%\src\condor_tests\*" "%testdir%\condor_tests\"
xcopy /Q "%GIT_HOME%\src\condor_tests\Check\*" "%testdir%\condor_tests\Check\"
xcopy /Q "%GIT_HOME%\src\condor_scripts\*.pm" "%testdir%\condor_tests\"
xcopy /Q "%GIT_HOME%\src\condor_scripts\batch_test.pl" "%testdir%\condor_tests\"
REM xcopy /Q "%GIT_HOME%\src\condor_examples\*" "%testdir%\condor_examples\"

devenv condor.sln /Build %bldtype% /Project BLD_TESTS
del /q "src\condor_tests\%bldtype%\sleep.*"
xcopy /Q "src\condor_tests\list_*"          "%testdir%\condor_tests"
xcopy /Q "src\condor_tests\%bldtype%\*.exe" "%testdir%\condor_tests"
xcopy /Q "src\condor_tests\%bldtype%\*.pdb" "%testdir%\condor_tests"

@echo >"%testdir%\setenv.bat" set PATH=%~sf1\bin;%%path%%
@echo >>"%testdir%\setenv.bat" set CONDOR_CONFIG=%~sf1\condor_config
@echo >>"%testdir%\setenv.bat" doskey m=mep $*
@echo >>"%testdir%\setenv.bat" doskey edit=devenv /edit $*
@echo >>"%testdir%\setenv.bat" doskey home=cd /d "%~f1"
@cd "%testdir%"
start cmd /k setenv.bat

goto :EOF

build_workspace.bat

@echo off
@setlocal
if not exist nmi_tools goto usage

if "%1"=="" goto usage
if "%1"=="-h" goto usage
if "%1"=="-help" goto usage
if "%1"=="/help" goto usage

set sha=%1
shift
if "%sha%"=="index" (
   for /F %%I in ('git write-tree') do (
      set sha=%%I
   )
)

set submit=batlab.chtc.wisc.edu
set tar=git archive %sha%
set platforms=all

set dir=wk%RANDOM%
set dir=%dir: =%
set arg1=%~1
if NOT "%arg1:~0,1%" == "-" (
   set desc=%arg1: =_%
   set dir=%dir%%arg1:~0,6%
   shift
)
set desc=%desc: =%

if NOT "%1"=="" set platforms=%1

@echo on
%tar% | plink -2 -i %USERPROFILE%\keys\putty.ppk -l %USERNAME% %submit% cd workspace ; mkdir %dir% ; cd %dir% ; tar xf - ; ../submit_workspace %desc% %platforms%
@echo off
goto :EOF

::


:usage
@echo This script must be run from the CONDOR_SRC directory
@echo Usage: %0 sha1 [description] [all ^| win ^| win+ ^| warn]
@echo.  send the given sha1 or branch to NMI submit machine as ~/workspace/wk^<rand^>
@echo.  and then run ~/workspace/build_workspace to build it.
@echo.  if sha1 is index, then git write-tree is used to generate the sha1

submit_workspace

#!/bin/bash

# args are [-test <runid>] <descrip> [<platforms>]
function usage
{
   echo "Usage: $0 [-test <runid>] <descrip> [<platforms>]"
   echo "   <platforms> may be comma separated list of NMI platforms or a category"
   echo "               categories are: all|win|fedora|deb|mac|warn|fast|win+"
}


declare -A plat
plat[win]="x86_64_Windows7,x86_64_Windows8,x86_64_Windows10"
plat[fedora]="x86_64_Fedora21,x86_64_Fedora22"
plat[deb]="x86_64_Debian6,x86_64_Debian7,x86_64_Debian8"
plat[mac]="x86_64_MacOSX7,x86_64_MacOSX8,x86_64_MacOSX10"
plat[fast]="${plat["fedora"]},x86_64_Solaris11,x86_64_Windows8"
plat[warn]="${plat["fedora"]}"
plat[win+]="${plat["win"]},${plat["fedora"]}"
plat[all]="all"

# if first arg is -test then second arg must be a runid
RUNID=""
TASK="building"
if [[ "$1" = "-test" ]]
then
   if [ -z "$2" ]; then usage; exit 1; fi
   RUNID="$2"
   TASK="testing"
   shift; shift;
fi

# next arg (description) is required
if [ -z "$1" ]; then usage; exit 1; fi
TAG="$1"

PLATFORMS="$2"
if [[ $2 = "all" || $2 = "win" || $2 = "fedora" || $2 = "deb" || $2 = "mac" || $2 = "fast" || $2 = "win+" ]]
then
    PLATFORMS="${plat[$2]}"
fi

pwd
if [ -d "nmi_tools" ]
then
   echo "$TASK temporary workspace"
   pushd nmi_tools
else
   echo "$TASK main workspace"
   pushd ~johnkn/workspace/CONDOR_SRC/nmi_tools
fi
pwd

if [ -z "$RUNID" ]
then
  #./condor_nmi_submit --build --git --notify-fail-only --workspace=.. --desc="$TAG" -platforms="$PLATFORMS"
  echo ./condor_nmi_submit --build --git --notify-fail-only --workspace=.. --desc=\"$TAG\" -platforms=\"$PLATFORMS\"
else
  #./condor_nmi_submit --test --buildid=$RUNID --workspace=.. --desc="$TAG" -platforms="$PLATFORMS" --ignore-missing-platforms --test-sources-from-workspace
  echo ./condor_nmi_submit --test --buildid=$RUNID --workspace=.. --desc=\"$TAG\" -platforms=\"$PLATFORMS\" --ignore-missing-platforms --test-sources-from-workspace
fi
popd