HTCondorWiki: Tjs Useful Classad Transforms

This page has useful classad tranform rules.

use by saving each as a file, and passing that file to condor_transform_ads like this:

condor_transform_ads -rules to_docker.xfm -in vanilla_jobs.ads -out docker_jobs.ads

Convert job to docker job

Invoke with Image=<image-name> to override default docker image

to_docker.xfm

Name VanillaToDocker
Requirements Universe == 5 && ! WantDocker

# invoker can pass image=<image> on command line to override...
If ! defined image
  image = SL6
Endif

# Convert to docker, rewrite executable to be relative to the sandbox
#
Set     WantDocker  = true
Set     DockerImage = "$F(image)"
Set     Cmd         = "./$Fnx(MY.Cmd)"
Rename  Requirements VanillaRequirements
Set     Requirements = Target.HasDocker

Set accounting group

Set accounting group based on username and preferred group. Assumes that the "groups" user map has been loaded.

set_accounting_group.xfm

Name SetAccountingGroup
# convert AccountingGroup into a preferred group, then map
# the user into an Accounting group honoring their preference
Rename  AccountingGroup PreferredGroup
Default PreferredGroup  undefined
EvalSet AccountingGroup mapUser("groups", Owner, PreferredGroup)

Simulate batlab execute nodes

Converts a set of STARTD ads from a single machine, into multiple startd ads, each with a different architecture and operating system. Invoke with

DOMAIN=x.y.z set the default domain for generated machine names
SharedFileSystem=true simulate a shared file system.

to_multi_platform.xfm

# Transform a each STARTD ad into multiple ads, each with a differnt OS and Arch
NAME Simulate multiple STARTD platforms
REQUIREMENTS MyType == "Machine"

# extract slotname from the ad name and manufacture a machine index
EVALMACRO SLOT = splitslotname(Name)[0]
INDEX = $(ROW)+1

# rewrite the machine name and domain
#
SET UidDomain = "$(DOMAIN:cs.wisc.edu)"
SET Machine = "exec$INT(INDEX,%02d).$(MY.UidDomain)"
if defined SLOT
  SET  Name = "$(SLOT)@$(My.Machine)"
else
  COPY Machine Name
endif
COPY Machine PrivateNetworkName

if $(SharedFileSystem)
  COPY UidDomain FileSystemDomain
else
  COPY Machine FileSystemDomain
endif

SET Arch           = "$(ARCH)"
SET OpSys          = "$(OS)"
SET OpSysAndVer    = "$(OSAndVer)"
SET OpSysVer       = "$(OSVer)"
SET OpSysName      = "$(OSName)"
SET OpSysMajorVer  = "$(OSMajor)"
SET OpSysShortName = "$(OSShort)"
SET OpSysLegacy    = "$(OSLegacy)"


TRANSFORM Arch,OS,OSAndVer,OSVer,OSName,OSMajor,OSShort,OSLegacy from (
  X86_64 LINUX   Debian7      700 Debian       7 Debian  LINUX
  X86_64 LINUX   Debian8      800 Debian       8 Debian  LINUX
  X86_64 LINUX   SL6          602 SL           6 SL      LINUX
  X86_64 LINUX   SL7          702 SL           7 SL      LINUX
  X86_64 LINUX   RedHat6      607 RedHat       6 RedHat  LINUX
  X86_64 LINUX   RedHat6      607 RedHat       6 RedHat  LINUX
  INTEL  LINUX   RedHat6      607 RedHat       6 RedHat  LINUX
  X86_64 LINUX   RedHat7      702 RedHat       7 RedHat  LINUX
  X86_64 WINDOWS WINDOWS601   601 Windows7   601 Win7    WINNT61
  X86_64 WINDOWS WINDOWS602   602 Windows8   602 Win8    WINNT62
  X86_64 WINDOWS WINDOWS1000 1000 Windows10 1000 Win10   WINNT100
  X86_64 OSX     MacOSX7      705 Lion         7 MacOSX  OSX
  X86_64 OSX     MacOSX8      805 MountainLion 8 MacOSX  OSX
  X86_64 OSX     MacOSX10    1001 ElCapitan   10 MacOSX  OSX
  X86_64 OSX     MacOSX10    1005 ElCapitan   10 MacOSX  OSX
  X86_64 LINUX   Ubuntu12    1204 Ubuntu      12 Ubuntu  LINUX
  X86_64 LINUX   Ubuntu14    1404 Ubuntu      14 Ubuntu  LINUX
  X86_64 LINUX   Fedora23    2300 Fedora      23 Fedora  LINUX
  X86_64 LINUX   Fedora24    2200 Fedora      22 Fedora  LINUX
  X86_64 FREEBSD FreeBSD10   1003 FreeBSD     10 FreeBSD FREEBSD
)

Generate simulated dynamic slots from a partitionable slot

Converts a partitionable STARTD ads from a single machine, into a partitionable slot ad and multiple dynamic slot ads. It does NOT properly adjust the partitionable slot ad to account for resources consumed by the dynamic slot ads. Invoke with

DOMAIN=x.y.z set the default domain for generated machine names
User=bob set the name of the simulated user for the dynamic slots
NumDynSlots=N control the number of dynamic slots
ReqMem=NN set the memory assigned to the dynamic slots
Update=N set the update sequence number

to_dyn_slots.xfm

# Simulate one or more active dynamic slots from an idle partitionable one
# The original partitionable slot is passed through untransfomed followed
# by N partitionable slots by the TRANSFORM N+1 statement
#

# this trick currently results in infinite loop... if they are self-references <sigh>
Client = $(Client:submit.chtc.wisc.edu)
User   = $(User:bob)
ReqMem = $(ReqMem:4000)
NumDynSlots = $(NumDynSlots:1)

#IpAddr = split(MyAddress,"<?>&")[0]
EVALMACRO AddrIp = $(My.AddressV1)[0].a
EVALMACRO AddrPort = $(My.AddressV1)[0].port
EVALMACRO Slot = splitslotname(Name)[0]

#CurrentTime = time()
#CurrentTime = $EVAL(CurrentTime)
EVALMACRO CurrentTime time()

SET MyCurrentTime = $(CurrentTime)
DELETE /^Updates.+/
if defined UPDATE
   SET UpdateSequenceNumber $(Update)
endif

if defined DOMAIN
  EVALMACRO Host = split("$(My.Machine)",".")[0]
  SET Machine "$(Host).$F(DOMAIN)"
  SET Name "$(Slot)@$(Host).$F(DOMAIN)"
endif

SET ClassAdLifetime 600

if $(Step)
  # Convert partitionable slot to dynamic slot
  #
  SET NAME  "$(Slot)_$(Step)@$(My.Machine)"
  DELETE PartitionableSlot
  DELETE NumDynamicSlots
  DELETE /^Child.+/
  SET ParentSlotId  $(My.SlotId)
  SET DynamicSlot   true
  SET SlotType      "Dynamic"
  SET SlotTypeId    -1
  SET Activity      "Busy"
  SET State         "Claimed"
  SET PublicClaimId "<$(AddrIp):$(AddrPort)>#$(CurrentTime)#$RANDOM_INTEGER(10000,20000,1)#..."

  # set info about the executing job and claim
  #
  SET     JobId           "$RANDOM_INTEGER(11100,22200,1).0"
  SET     JobUniverse     5
  EVALSET JobStart        time() - 60*16
  SET     NumPids         2
  SET     GlobalJobId     "$(Client)#$(MY.JobId)#$(CurrentTime)"
  SET     ImageSize       $RANDOM_INTEGER(100000,1000000,128)
  SET     ExecutableSize  3
  EVALSET ResidentSetSize ImageSize + $RANDOM_INTEGER(1000,10000,128)
  COPY    JobStart EnteredCurrentState
  COPY    JobStart EnteredCurrentActivity
  EVALSET TotalClaimRunTime time() - EnteredCurrentState
  COPY    TotalClaimRunTime TotalJobRunTime
  COPY    TotalClaimRunTime TotalTimeClaimedBusy
  SET     TotalTimeClaimedIdle 2
  DELETE  TotalTimeDrainedRetiring
  DELETE  TotalTimeUnclaimedIdle
  DELETE  TotalTimeUnclaimedBenchmarking
  EVALSET RetirementTimeRemaining 60*60*24*3 - TotalClaimRunTime

  # set ownership
  #
  SET NiceUser      false
  SET RemoteOwner   "$(User)@chtc.wisc.edu"
  COPY RemoteOwner  RemoteUser
  SET RemoteNegotiatingGroup "<none>"
  SET RemoteAutoregroup false
  SET ClientMachine "$(Client)"

  # set resource usage
  #
  SET     Cpus      1
  COPY    Cpus      TotalSlotCpus
  SET     RequestMemory $(ReqMem)
  EVALSET Memory    quantize($(ReqMem),256)
  COPY    Memory    TotalSlotMemory
  SET     MemoryUsage ((ResidentSetSize + 1023)/1024)
  EVALSET Disk      Disk/40.0
  COPY    Disk      TotalSlotDisk


else

  # Pass original Partitionable Slot through unmolested at step 0
  # Should maybe subtract resource usage from it?


endif

TRANSFORM 1+$(NumDynSlots:1)