#!/usr/bin/python
# This serializes hourly/daily/weekly/monthly execution of cronjobs
# in vserver guests to avoid high load due to massive parallel
# execution of tasks (log file rotation, compression, service
# restarts). The program will only invoke the cron scripts if
# they are not invoked by separate cron process within the guest
# itself.
# Parameters:
# * --Test: Start script in test mode, this increases the
#   script's verbosity and blocks real execution of cron tasks
#   in the guests. 
# * --JobType:
# * --Verbose: Run script in verbose mode to get more output
# * --MaxParallel: Maximal number of parallel guest cron job
#   invocations at a time.
# * --CompletionWaitTime: Time to wait for guest cron scripts
#   to complete before attemting to start another execution process.

import os
from stat import S_ISDIR
import subprocess
import sys
import threading

class JoinableSubprocess(object):
  def __init__(self, vserverName, cmd):
    self.vserverName=vserverName
    self.cmd=cmd
    self.process=subprocess.Popen(self.cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    self.output=None
    self.thread=threading.Thread(target=self.doCommunicate)
    self.thread.start()

  def doCommunicate(self):
    self.output=self.process.communicate()

  def hasFinished(self):
    if self.thread.is_alive(): return False
    if (self.process.returncode != 0) or (self.output[1] != ''):
      print >>sys.stderr, "Abnormal job termination in %s with status %s, stdout: %s, stderr %s" % (self.vserverName, self.process.returncode, self.output[0], self.output[1])
    return True

maxParallelProcesses=2
currentProcessList=[]
completionWaitTime=10
cronJobType=None
testModeFlag=False
verboseModeFlag=False
maxProcessesWarningIssuedFlag=False

argPos=1
while argPos < len(sys.argv):
  argName=sys.argv[argPos]
  argPos=argPos+1

  if "--CompletionWaitTime" == argName:
    completionWaitTime=int(sys.argv[argPos])
    argPos=argPos+1
    continue
  if "--JobType" == argName:
    cronJobType=sys.argv[argPos]
    argPos=argPos+1
    if not cronJobType in ['hourly', 'daily', 'weekly', 'monthly']:
      print >>sys.stderr, "Job type must be one of 'hourly', 'daily', 'weekly', 'monthly'"
      sys.exit(1)
    continue
  if "--MaxParallel" == argName:
    maxParallelProcesses=int(sys.argv[argPos])
    argPos=argPos+1
    continue
  if "--Test" == argName:
    testModeFlag=True
    continue
  if "--Verbose" == argName:
    verboseModeFlag=True
    continue
  print >>sys.stderr, "Unsupported argument \"%s\"" % argName
  sys.exit(1)

if cronJobType == None:
  print >>sys.stderr, "No job type given"
  sys.exit(1)


# Now loop over all vservers and check if they meet the preconditions
vserverNameList=os.listdir('/etc/vservers')
while (len(vserverNameList) != 0) or (len(currentProcessList) != 0):

# See if process limit just reached. If yes wait for any process
# to complete
  procPos=len(currentProcessList)-1
  while procPos >= 0:
    joinableProcess=currentProcessList[procPos]
    if joinableProcess.hasFinished():
      if verboseModeFlag:
        print >>sys.stderr, "VserverCronSerialisation guest %s job %s completed" % (joinableProcess.vserverName, cronJobType)

      currentProcessList=currentProcessList[:procPos]+currentProcessList[procPos+1:]
    procPos=procPos-1
  if len(currentProcessList) >= maxParallelProcesses:
    if not maxProcessesWarningIssuedFlag:
      print >>sys.stderr, "VserverCronSerialisation: process limit of %s reached, consider increasing it" % maxParallelProcesses
      maxProcessesWarningIssuedFlag=True
    currentProcessList[0].thread.join(1)
    continue

# Just wait for remaining processes if already all of them started
  if len(vserverNameList) == 0:
    if len(currentProcessList) == 0: break
    currentProcessList[0].thread.join(1)
    continue

  vserverName=vserverNameList[0]
  vserverNameList=vserverNameList[1:]
  if vserverName[0] == '.': continue
  vserverConfDirName=os.path.join('/etc/vservers', vserverName)
  mode=os.stat(vserverConfDirName).st_mode
  if not S_ISDIR(mode): continue

# Shell version checked also if 'NAME' is not empty string, not clear
# why that was used.
  vserverState=subprocess.check_output(['/usr/sbin/vserver-info', vserverName, 'RUNNING'])
  if vserverState == '0\n': continue
  if vserverState != '1\n':
    print >>sys.stderr, "Unexpected vserver state for %s: %s" % (vserverName, vserverState[:len(vserverState)-1])

  vserverBaseDir=os.path.join(vserverConfDirName, 'vdir')
  if (not os.path.isfile(os.path.join(vserverBaseDir, 'usr/sbin/cron'))) or (not os.path.isdir(os.path.join(vserverBaseDir, 'etc/cron.'+cronJobType))):
    if verboseModeFlag:
      print >>sys.stderr, "Vserver %s has no cron installation, ignored" % vserverName
    continue

# So cron is installed and should be functional. See if it contains
# separate crontab configuration
  cronGuestSupportOutput=None
  try:
    cronGuestSupportOutput=subprocess.check_output(['grep', '-q', '-E', '-e', "run-parts --report /etc/cron\\.%s" % cronJobType, os.path.join(vserverBaseDir, 'etc/crontab')])
  except subprocess.CalledProcessError as e:
    if e.returncode != 1:
      print >>sys.stderr, "Failed to check guest %s crontab: %s", (vserverName, e)

  if cronGuestSupportOutput == '':
    if verboseModeFlag:
      print >>sys.stderr, "Vserver %s contains separate cron cronfiguration, serialisation omitted" % vserverName
    continue

  if testModeFlag:
    print >>sys.stderr, "VserverCronSerialisation: Normal run would start %s type %s jobs" % (vserverName, cronJobType)
    continue

  if verboseModeFlag:
    print >>sys.stderr, "VserverCronSerialisation: Starting guest %s jobs from /etc/cron.%s" % (vserverName, cronJobType)


# Start the next process
  joinableProcess=JoinableSubprocess(vserverName, ['vserver', vserverName, 'exec', '/bin/sh', '-e', '-c', "cd /; run-parts --report /etc/cron.%s" % cronJobType])
  currentProcessList+=[joinableProcess]
  joinableProcess.thread.join(completionWaitTime)

if verboseModeFlag:
  print >>sys.stderr, "VserverCronSerialisation %s completed" % cronJobType