diff --git a/helix-control-m/101-create-first-job-flow/AutomationAPISampleFlow.json b/helix-control-m/101-create-first-job-flow/AutomationAPISampleFlow.json deleted file mode 100644 index 609011c..0000000 --- a/helix-control-m/101-create-first-job-flow/AutomationAPISampleFlow.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "Defaults" : { - "Application" : "SampleApp", - "SubApplication" : "SampleSubApp", - "RunAs" : "USERNAME", - "Host" : "HOST", - - "Job": { - "When" : { - "Months": ["JAN", "OCT", "DEC"], - "MonthDays":["22","1","11"], - "WeekDays":["MON","TUE", "WED", "THU", "FRI"], - "FromTime":"0300", - "ToTime":"2100" - }, - "ActionIfFailure" : { - "Type": "If", - "CompletionStatus": "NOTOK", - - "mailToTeam": { - "Type": "Mail", - "Message": "%%JOBNAME failed", - "To": "team@mycomp.com" - } - } - } - }, - - "AutomationAPISampleFlow": { - "Type": "Folder", - "Comment" : "Code reviewed by John", - - "CommandJob": { - "Type": "Job:Command", - "Command": "echo my 1st job" - }, - - "ScriptJob": { - "Type": "Job:Script", - "FilePath":"SCRIPT_PATH", - "FileName":"SCRIPT_NAME" - }, - - "Flow": { - "Type": "Flow", - "Sequence": ["CommandJob", "ScriptJob"] - } - } -} diff --git a/helix-control-m/101-create-first-job-flow/README.md b/helix-control-m/101-create-first-job-flow/README.md deleted file mode 100644 index dc02df0..0000000 --- a/helix-control-m/101-create-first-job-flow/README.md +++ /dev/null @@ -1,15 +0,0 @@ -## Creating Your First Job Flow - -Tutorial on the [product web page](https://docs.bmc.com/docs/display/ctmSaaSAPI/Creating+your+first+job+flow) -that explains how to write jobs that execute OS commands and scripts. - -```javascript -"CommandJob": { - "Type": "Job:Command", - "Host": "HOST", - "Command": "echo my 1st job" -} -``` - -See the [Automation API - Services](https://docs.bmc.com/docs/display/ctmSaaSAPI/Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/ctmSaaSAPI/Code+Reference) documentation for more information. diff --git a/helix-control-m/101-running-file-transfer-and-database-query-job-flow/AutomationAPIFileTransferDatabaseSampleFlow.json b/helix-control-m/101-running-file-transfer-and-database-query-job-flow/AutomationAPIFileTransferDatabaseSampleFlow.json deleted file mode 100644 index 7d69e3b..0000000 --- a/helix-control-m/101-running-file-transfer-and-database-query-job-flow/AutomationAPIFileTransferDatabaseSampleFlow.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "Defaults" : { - "Application" : "SampleApp", - "SubApplication" : "SampleSubApp", - "Host" : "HOST", - "Centralized": true, - "Variables": [ - {"DestDataFile": "DESTINATION_FILE"}, - {"SrcDataFile": "SOURCE_FILE"} - ], - "Job": { - "When" : { - "FromTime":"0300", - "ToTime":"2100" - } - } - }, - - "SFTP-CP": { - "Type": "ConnectionProfile:FileTransfer:SFTP", - "HostName": "SFTP_SERVER", - "Port": "22", - "User" : "SFTP_USER", - "Password" : "SFTP_PASSWORD" - }, - - "Local-CP" : { - "Type" : "ConnectionProfile:FileTransfer:Local", - "User" : "USER", - "Password" : "PASSWORD" - }, - - "DB-CP": { - "Type": "ConnectionProfile:Database:PostgreSQL", - "Host": "DATABASE_SERVER", - "Port":"5432", - "User": "DATABASE_USER", - "Password": "DATABASE_PASSWORD", - "DatabaseName": "postgres" - }, - - "AutomationAPIFileTransferDatabaseSampleFlow": { - "Type": "Folder", - "Comment" : "Code reviewed by John", - - "GetData": { - "Type" : "Job:FileTransfer", - "ConnectionProfileSrc" : "SFTP-CP", - "ConnectionProfileDest" : "Local-CP", - - "FileTransfers" : - [ - { - "Src" : "%%SrcDataFile", - "Dest": "%%DestDataFile", - "TransferOption": "SrcToDest", - "TransferType": "Binary", - "PreCommandDest": { - "action": "rm", - "arg1": "%%DestDataFile" - }, - "PostCommandDest": { - "action": "chmod", - "arg1": "700", - "arg2": "%%DestDataFile" - } - } - ] - }, - - "UpdateRecords": { - "Type": "Job:Database:SQLScript", - "SQLScript": "/home/USER/automation-api-quickstart/helix-control-m/101-running-file-transfer-and-database-query-job-flow/processRecords.sql", - "ConnectionProfile": "DB-CP" - }, - - "Flow": { - "Type": "Flow", - "Sequence": ["GetData", "UpdateRecords"] - } - } -} diff --git a/helix-control-m/101-running-file-transfer-and-database-query-job-flow/README.md b/helix-control-m/101-running-file-transfer-and-database-query-job-flow/README.md deleted file mode 100644 index 697202b..0000000 --- a/helix-control-m/101-running-file-transfer-and-database-query-job-flow/README.md +++ /dev/null @@ -1,39 +0,0 @@ -## Running File Transfer and Database queries job flow - -Tutorial on the [product web page](https://docs.bmc.com/docs/display/ctmSaaSAPI/Running+applications+and+programs+in+your+environment) that explains how to write jobs that execute File Transfer and Database queries. - -```javascript -"GetData": { - "Type" : "Job:FileTransfer", - "ConnectionProfileSrc" : "SFTP-CP", - "ConnectionProfileDest" : "Local-CP", - - "FileTransfers" : - [ - { - "Src" : "%%SrcDataFile", - "Dest": "%%DestDataFile", - "TransferOption": "SrcToDest", - "TransferType": "Binary", - "PreCommandDest": { - "action": "rm", - "arg1": "%%DestDataFile" - }, - "PostCommandDest": { - "action": "chmod", - "arg1": "700", - "arg2": "%%DestDataFile" - } - } - ] -}, - -"UpdateRecords": { - "Type": "Job:Database:SQLScript", - "SQLScript": "/home//automation-api-quickstart/helix-control-m/101-running-file-transfer-and-database-query-job-flow/processRecords.sql", - "ConnectionProfile": "DB-CP" -} -``` - -See the [Automation API - Services](https://docs.bmc.com/docs/display/ctmSaaSAPI/Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/ctmSaaSAPI/Code+Reference) documentation for more information. diff --git a/helix-control-m/101-running-file-transfer-and-database-query-job-flow/processRecords.sql b/helix-control-m/101-running-file-transfer-and-database-query-job-flow/processRecords.sql deleted file mode 100644 index 43338e2..0000000 --- a/helix-control-m/101-running-file-transfer-and-database-query-job-flow/processRecords.sql +++ /dev/null @@ -1 +0,0 @@ -select 'Parameter'; \ No newline at end of file diff --git a/helix-control-m/101-running-hadoop-spark-job-flow/AutomationAPISampleHadoopFlow.json b/helix-control-m/101-running-hadoop-spark-job-flow/AutomationAPISampleHadoopFlow.json deleted file mode 100644 index dc45b51..0000000 --- a/helix-control-m/101-running-hadoop-spark-job-flow/AutomationAPISampleHadoopFlow.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "Defaults" : { - "Application": "SampleApp", - "SubApplication": "SampleSubApp", - "Host" : "HOST", - "Job": { - "When" : { - "FromTime":"0300", - "ToTime":"2100" - } - }, - "Job:Hadoop" : { - "ConnectionProfile": "SAMPLE_CP" - } - }, - - "SAMPLE_CP" : - { - "Type" : "ConnectionProfile:Hadoop", - "Centralized" : true - }, - - "AutomationAPIHadoopSampleFlow": { - "Type": "Folder", - "Comment" : "Code reviewed by John", - - "ProcessData": { - "Type": "Job:Hadoop:Spark:Python", - "SparkScript": "file:///home/USER/automation-api-quickstart/helix-control-m/101-running-hadoop-spark-job-flow/processData.py", - - "Arguments": [ - "file:///home/USER/automation-api-quickstart/helix-control-m/101-running-hadoop-spark-job-flow/processData.py", - "file:///home/USER/automation-api-quickstart/helix-control-m/101-running-hadoop-spark-job-flow/processDataOutDir" - ], - - "PreCommands" : { - "Commands" : [ - { "rm":"-R -f file:///home/USER/automation-api-quickstart/helix-control-m/101-running-hadoop-spark-job-flow/processDataOutDir" } - ] - } - }, - "CopyOutputData" : - { - "Type" : "Job:Hadoop:HDFSCommands", - "Commands" : [ - {"rm" : "-R -f samplesOut" }, - {"mkdir" : "samplesOut" }, - {"cp" : "file:///home/USER/automation-api-quickstart/helix-control-m/101-running-hadoop-spark-job-flow/* samplesOut" } - ] - }, - - "DataProcessingFlow": { - "Type": "Flow", - "Sequence": ["ProcessData","CopyOutputData"] - } - } -} diff --git a/helix-control-m/101-running-hadoop-spark-job-flow/README.md b/helix-control-m/101-running-hadoop-spark-job-flow/README.md deleted file mode 100644 index c3026da..0000000 --- a/helix-control-m/101-running-hadoop-spark-job-flow/README.md +++ /dev/null @@ -1,24 +0,0 @@ -## Create Your First Hadoop Job Flow - -Tutorial on the [product web page](https://docs.bmc.com/docs/display/ctmSaaSAPI/Running+applications+and+programs+in+your+environment) that explains how to write jobs that execute Spark and HDFS commands. - -```javascript -"ProcessData": { - "Type": "Job:Hadoop:Spark:Python", - "SparkScript": "file:///home/[USER]/automation-api-quickstart/helix-control-m/101-running-hadoop-spark-job-flow/processData.py", - - "Arguments": [ - "file:///home/[USER]/automation-api-quickstart/helix-control-m/101-running-hadoop-spark-job-flow/processData.py", - "file:///home/[USER]/automation-api-quickstart/helix-control-m/101-running-hadoop-spark-job-flow/processDataOutDir" - ], - - "PreCommands" : { - "Commands" : [ - { "rm":"-R -f file:///home/[USER]/automation-api-quickstart/helix-control-m/101-running-hadoop-spark-job-flow/processDataOutDir" } - ] - } -} -``` - -See the [Automation API - Services](https://docs.bmc.com/docs/display/ctmSaaSAPI/Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/ctmSaaSAPI/Code+Reference) documentation for more information. diff --git a/helix-control-m/101-running-hadoop-spark-job-flow/processData.py b/helix-control-m/101-running-hadoop-spark-job-flow/processData.py deleted file mode 100644 index f8f0b29..0000000 --- a/helix-control-m/101-running-hadoop-spark-job-flow/processData.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import print_function - -import sys - -from pyspark import SparkContext - - -inputFile = sys.argv[1] -outputDir = sys.argv[2] - -sc = SparkContext(appName="processDataSample") - -text_file = sc.textFile(inputFile) - -counts = text_file.flatMap(lambda line: line.split(" ")) \ - .map(lambda word: (word, 1)) \ - .reduceByKey(lambda a, b: a + b) - -counts.saveAsTextFile(outputDir) \ No newline at end of file diff --git a/helix-control-m/101-running-script-command-job-flow/AutomationAPISampleFlow.json b/helix-control-m/101-running-script-command-job-flow/AutomationAPISampleFlow.json deleted file mode 100644 index faff4f6..0000000 --- a/helix-control-m/101-running-script-command-job-flow/AutomationAPISampleFlow.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "Defaults" : { - "Application" : "SampleApp", - "SubApplication" : "SampleSubApp", - "RunAs" : "USERNAME", - "Host" : "HOST", - - "Job": { - "When" : { - "Months": ["JAN", "OCT", "DEC"], - "MonthDays":["22","1","11"], - "WeekDays":["MON","TUE", "WED", "THU", "FRI"], - "FromTime":"0300", - "ToTime":"2100" - }, - "ActionIfFailure" : { - "Type": "If", - "CompletionStatus": "NOTOK", - - "mailToTeam": { - "Type": "Mail", - "Message": "%%JOBNAME failed", - "To": "team@mycomp.com" - } - } - } - }, - - "AutomationAPISampleFlow": { - "Type": "Folder", - "Comment" : "Code reviewed by John", - - "CommandJob": { - "Type": "Job:Command", - "Command": "COMMAND" - }, - - "ScriptJob": { - "Type": "Job:Script", - "FilePath":"SCRIPT_PATH", - "FileName":"SCRIPT_NAME" - }, - - "Flow": { - "Type": "Flow", - "Sequence": ["CommandJob", "ScriptJob"] - } - } -} diff --git a/helix-control-m/101-running-script-command-job-flow/README.md b/helix-control-m/101-running-script-command-job-flow/README.md deleted file mode 100644 index 1ec3ba8..0000000 --- a/helix-control-m/101-running-script-command-job-flow/README.md +++ /dev/null @@ -1,19 +0,0 @@ -## Running Script, Program and Command job flow - -Tutorial on the [product web page](https://docs.bmc.com/docs/display/ctmSaaSAPI/Running+applications+and+programs+in+your+environment) that explains how to write jobs that execute Scripts, Programs and Commands. - -```javascript -"CommandJob": { - "Type": "Job:Command", - "Command": "COMMAND" -}, - -"ScriptJob": { - "Type": "Job:Script", - "FilePath":"SCRIPT_PATH", - "FileName":"SCRIPT_NAME" -} -``` - -See the [Automation API - Services](https://docs.bmc.com/docs/display/ctmSaaSAPI/Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/ctmSaaSAPI/Code+Reference) documentation for more information. diff --git a/helix-control-m/102-automate-code-deployment/AutomationAPISampleFlow.json b/helix-control-m/102-automate-code-deployment/AutomationAPISampleFlow.json deleted file mode 100644 index 7aeb5a3..0000000 --- a/helix-control-m/102-automate-code-deployment/AutomationAPISampleFlow.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "Defaults" : { - "Application" : "SampleApp", - "SubApplication" : "SampleSubApp", - "RunAs" : "USERNAME", - - "Job": { - "When" : { - "Months": ["JAN", "OCT", "DEC"], - "MonthDays":["22","1","11"], - "WeekDays":["MON","TUE", "WED", "THU", "FRI"], - "FromTime":"0300", - "ToTime":"2100" - }, - "ActionIfFailure" : { - "Type": "If", - "CompletionStatus": "NOTOK", - - "mailToTeam": { - "Type": "Mail", - "Message": "%%JOBNAME failed", - "To": "team@mycomp.com" - } - } - } - }, - - "AutomationAPISampleFlow": { - "Type": "Folder", - "Comment" : "Code reviewed by John", - - "CommandJob": { - "Type": "Job:Command", - "Command": "echo my 1st job", - "Host" : "host1" - }, - - "ScriptJob": { - "Type": "Job:Script", - "FilePath":"SCRIPT_PATH", - "FileName":"SCRIPT_NAME", - "Host" : "host2" - }, - - "Flow": { - "Type": "Flow", - "Sequence": ["CommandJob", "ScriptJob"] - } - } -} diff --git a/helix-control-m/102-automate-code-deployment/DeployCode.py b/helix-control-m/102-automate-code-deployment/DeployCode.py deleted file mode 100644 index b757146..0000000 --- a/helix-control-m/102-automate-code-deployment/DeployCode.py +++ /dev/null @@ -1,20 +0,0 @@ -import requests # pip install requests if you don't have it already -import urllib3 - -urllib3.disable_warnings() # disable warnings when creating unverified requests - -endPoint = 'https:///automation-api' -token= - -# ----------------- -# Built -uploaded_files = [ - ('definitionsFile', ('Jobs.json', open('c:\\src\Jobs.json', 'rb'), 'application/json')) -] - -r = requests.post(endPoint + '/deploy', files=uploaded_files, headers={'x-api-key': + token}, verify=False) - -print(r.content) -print(r.status_code) - -exit(r.status_code == requests.codes.ok) diff --git a/helix-control-m/102-automate-code-deployment/DeployCode.sh b/helix-control-m/102-automate-code-deployment/DeployCode.sh deleted file mode 100644 index 851fcf2..0000000 --- a/helix-control-m/102-automate-code-deployment/DeployCode.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -for f in *.json; do - echo "Deploying file $f"; - ctm deploy $f -e ciEnvironment; -done diff --git a/helix-control-m/102-automate-code-deployment/DeployDescriptor.json b/helix-control-m/102-automate-code-deployment/DeployDescriptor.json deleted file mode 100644 index 8fda9fc..0000000 --- a/helix-control-m/102-automate-code-deployment/DeployDescriptor.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "DeployDescriptor": - [ - { - "Comment": "Set run as user in Defaults to the prod automation user", - "ApplyOn": { - "@":"Defaults" - }, - "Property" :"RunAs", - "Assign" : "workbench" - }, - { - "Comment": "Modify Application property to comply with Development environment", - "Property" :"Application", - "Replace" : [ {"(.*)" : "Dev$1"} ] - }, - { - "Comment": "Distribute jobs across hosts available in Development environment based on job names", - "Property": "Host", - "Source": "@", - "Replace": [ - { "Command.*" : "workbench"}, - { "Script.*" : "workbench"} - ] - } - ] -} diff --git a/helix-control-m/102-automate-code-deployment/README.md b/helix-control-m/102-automate-code-deployment/README.md deleted file mode 100644 index 1b23f82..0000000 --- a/helix-control-m/102-automate-code-deployment/README.md +++ /dev/null @@ -1,17 +0,0 @@ -## How to Automate Code Deployment - -Tutorial on the [product web page](https://docs.bmc.com/docs/display/ctmSaaSAPI/Automating+code+deployment) -that explains how DevOps engineer can automate code deployment. - -The bash and python examples can be used as a script of a **Jenkins** step that pushes Git changes to Control-M. - -```bash -#!/bin/bash -for f in *.json; do - echo "Deploying file $f"; - ctm deploy $f -e ciEnvironment; -done -``` - -See the [Automation API - Services](https://docs.bmc.com/docs/display/ctmSaaSAPI/Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/ctmSaaSAPI/Code+Reference) documentation for more information. diff --git a/helix-control-m/102-automate-code-deployment/ciEnvironmentJobs.json b/helix-control-m/102-automate-code-deployment/ciEnvironmentJobs.json deleted file mode 100644 index cffd11a..0000000 --- a/helix-control-m/102-automate-code-deployment/ciEnvironmentJobs.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "AutomationAPISampleFlow": { - "Type": "Folder", - "ControlmServer": "IN01", - "RunAs": "USERNAME", - "SubApplication": "SampleSubApp", - "Application": "SampleApp", - "CommandJob": { - "Type": "Job:Command", - "SubApplication": "SampleSubApp", - "Host": "host1", - "RunAs": "USERNAME", - "Application": "SampleApp", - "Command": "echo my 1st job", - "When": { - "WeekDays": [ - "MON", - "TUE", - "WED", - "THU", - "FRI" - ], - "Months": [ - "JAN", - "OCT", - "DEC" - ], - "MonthDays": [ - "1", - "11", - "22" - ], - "ToTime": "2100", - "FromTime": "0300" - }, - "IfBase:Folder:CompletionStatus_0": { - "Type": "If:CompletionStatus", - "CompletionStatus": "NOTOK", - "Mail_0": { - "Type": "Action:Mail", - "To": "team@mycomp.com", - "Message": "%%JOBNAME failed" - } - } - }, - "ScriptJob": { - "Type": "Job:Script", - "SubApplication": "SampleSubApp", - "FileName": "SCRIPT_NAME", - "Host": "host2", - "FilePath": "SCRIPT_PATH", - "RunAs": "USERNAME", - "Application": "SampleApp", - "When": { - "WeekDays": [ - "MON", - "TUE", - "WED", - "THU", - "FRI" - ], - "Months": [ - "JAN", - "OCT", - "DEC" - ], - "MonthDays": [ - "1", - "11", - "22" - ], - "ToTime": "2100", - "FromTime": "0300" - }, - "IfBase:Folder:CompletionStatus_0": { - "Type": "If:CompletionStatus", - "CompletionStatus": "NOTOK", - "Mail_0": { - "Type": "Action:Mail", - "To": "team@mycomp.com", - "Message": "%%JOBNAME failed" - } - } - }, - "Flow": { - "Type": "Flow", - "Sequence": [ - "CommandJob", - "ScriptJob" - ] - } - }, - "JobsRunInDockerSample": { - "Type": "Folder", - "ControlmServer": "IN01", - "RunAs": "controlm", - "SubApplication": "SampleSubApp", - "Application": "SampleApp", - "CommandJob": { - "Type": "Job:Command", - "SubApplication": "SampleSubApp", - "Host": "yosi_gal", - "RunAs": "controlm", - "Application": "SampleApp", - "Command": "whoami ; pwd; ls -l" - } - }, - "AutomationAPIFileTransferDatabaseSampleFlow": { - "Type": "Folder", - "Variables": [ - { - "DestDataFile": "DESTINATION_FILE" - }, - { - "SrcDataFile": "SOURCE_FILE" - } - ], - "ControlmServer": "IN01", - "SubApplication": "SampleSubApp", - "Application": "SampleApp", - "GetData": { - "Type": "Job:FileTransfer", - "ConnectionProfileSrc": "SFTP-CP", - "ConnectionProfileDest": "Local-CP", - "SubApplication": "SampleSubApp", - "Host": "HOST", - "RunAs": "SFTP-CP+Local-CP", - "Application": "SampleApp", - "Variables": [ - { - "DestDataFile": "DESTINATION_FILE" - }, - { - "SrcDataFile": "SOURCE_FILE" - } - ], - "FileTransfers": [ - { - "ABSTIME": "0", - "VERNUM": "0", - "Dest": "%%DestDataFile", - "SRCOPT": "0", - "TransferType": "Binary", - "CASEIFS": "0", - "DSTOPT": "0", - "RECURSIVE": "0", - "TransferOption": "SrcToDest", - "Src": "%%SrcDataFile", - "TIMELIMIT": "0", - "EXCLUDE_WILDCARD": "0", - "NULLFLDS": "0", - "TRIM": "1", - "IF_EXIST": "0", - "UNIQUE": "0", - "PostCommandDest": { - "action": "chmod", - "arg2": "%%DestDataFile", - "arg1": "700" - }, - "PreCommandDest": { - "arg1": "%%DestDataFile", - "action": "rm" - } - } - ], - "When": { - "ToTime": "2100", - "FromTime": "0300" - } - }, - "UpdateRecords": { - "Type": "Job:Database:SQLScript", - "SQLScript": "/home/USER/automation-api-quickstart/101-running-file-transfer-and-database-query-job-flow/processRecords.sql", - "ConnectionProfile": "DB-CP", - "SubApplication": "SampleSubApp", - "Host": "HOST", - "RunAs": "DB-CP", - "Application": "SampleApp", - "Variables": [ - { - "DestDataFile": "DESTINATION_FILE" - }, - { - "SrcDataFile": "SOURCE_FILE" - } - ], - "When": { - "ToTime": "2100", - "FromTime": "0300" - } - }, - "Flow": { - "Type": "Flow", - "Sequence": [ - "GetData", - "UpdateRecords" - ] - } - }, - "JobsRunInDockerSample2": { - "Type": "Folder", - "ControlmServer": "IN01", - "RunAs": "controlm", - "SubApplication": "SampleSubApp", - "Application": "SampleApp", - "CommandJob": { - "Type": "Job:Command", - "SubApplication": "SampleSubApp", - "Host": "yosi_gal", - "RunAs": "controlm", - "Application": "SampleApp", - "Command": "whoami ; pwd; ls -l" - } - } -} diff --git a/helix-control-m/102-automate-code-deployment/transformed.json b/helix-control-m/102-automate-code-deployment/transformed.json deleted file mode 100644 index f43450e..0000000 --- a/helix-control-m/102-automate-code-deployment/transformed.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "AutomationAPISampleFlow": { - "Type": "Folder", - "ControlmServer": "IN01", - "RunAs": "USERNAME", - "SubApplication": "SampleSubApp", - "Application": "DevSampleApp", - "CommandJob": { - "Type": "Job:Command", - "SubApplication": "SampleSubApp", - "Host": "workbench", - "RunAs": "USERNAME", - "Application": "DevSampleApp", - "Command": "echo my 1st job", - "When": { - "WeekDays": [ - "MON", - "TUE", - "WED", - "THU", - "FRI" - ], - "Months": [ - "JAN", - "OCT", - "DEC" - ], - "MonthDays": [ - "1", - "11", - "22" - ], - "ToTime": "2100", - "FromTime": "0300" - }, - "IfBase:Folder:CompletionStatus_0": { - "Type": "If:CompletionStatus", - "CompletionStatus": "NOTOK", - "Mail_0": { - "Type": "Action:Mail", - "To": "team@mycomp.com", - "Message": "%%JOBNAME failed" - } - } - }, - "ScriptJob": { - "Type": "Job:Script", - "SubApplication": "SampleSubApp", - "FileName": "SCRIPT_NAME", - "Host": "workbench", - "FilePath": "SCRIPT_PATH", - "RunAs": "USERNAME", - "Application": "DevSampleApp", - "When": { - "WeekDays": [ - "MON", - "TUE", - "WED", - "THU", - "FRI" - ], - "Months": [ - "JAN", - "OCT", - "DEC" - ], - "MonthDays": [ - "1", - "11", - "22" - ], - "ToTime": "2100", - "FromTime": "0300" - }, - "IfBase:Folder:CompletionStatus_0": { - "Type": "If:CompletionStatus", - "CompletionStatus": "NOTOK", - "Mail_0": { - "Type": "Action:Mail", - "To": "team@mycomp.com", - "Message": "%%JOBNAME failed" - } - } - }, - "Flow": { - "Type": "Flow", - "Sequence": [ - "CommandJob", - "ScriptJob" - ] - } - }, - "JobsRunInDockerSample": { - "Type": "Folder", - "ControlmServer": "IN01", - "RunAs": "controlm", - "SubApplication": "SampleSubApp", - "Application": "DevSampleApp", - "CommandJob": { - "Type": "Job:Command", - "SubApplication": "SampleSubApp", - "Host": "workbench", - "RunAs": "controlm", - "Application": "DevSampleApp", - "Command": "whoami ; pwd; ls -l" - } - }, - "AutomationAPIFileTransferDatabaseSampleFlow": { - "Type": "Folder", - "Variables": [ - { - "DestDataFile": "DESTINATION_FILE" - }, - { - "SrcDataFile": "SOURCE_FILE" - } - ], - "ControlmServer": "IN01", - "SubApplication": "SampleSubApp", - "Application": "DevSampleApp", - "GetData": { - "Type": "Job:FileTransfer", - "ConnectionProfileSrc": "SFTP-CP", - "ConnectionProfileDest": "Local-CP", - "SubApplication": "SampleSubApp", - "Host": "HOST", - "RunAs": "SFTP-CP+Local-CP", - "Application": "DevSampleApp", - "Variables": [ - { - "DestDataFile": "DESTINATION_FILE" - }, - { - "SrcDataFile": "SOURCE_FILE" - } - ], - "FileTransfers": [ - { - "ABSTIME": "0", - "VERNUM": "0", - "Dest": "%%DestDataFile", - "SRCOPT": "0", - "TransferType": "Binary", - "CASEIFS": "0", - "DSTOPT": "0", - "RECURSIVE": "0", - "TransferOption": "SrcToDest", - "Src": "%%SrcDataFile", - "TIMELIMIT": "0", - "EXCLUDE_WILDCARD": "0", - "NULLFLDS": "0", - "TRIM": "1", - "IF_EXIST": "0", - "UNIQUE": "0", - "PostCommandDest": { - "action": "chmod", - "arg2": "%%DestDataFile", - "arg1": "700" - }, - "PreCommandDest": { - "arg1": "%%DestDataFile", - "action": "rm" - } - } - ], - "When": { - "ToTime": "2100", - "FromTime": "0300" - } - }, - "UpdateRecords": { - "Type": "Job:Database:SQLScript", - "SQLScript": "/home/USER/automation-api-quickstart/101-running-file-transfer-and-database-query-job-flow/processRecords.sql", - "ConnectionProfile": "DB-CP", - "SubApplication": "SampleSubApp", - "Host": "HOST", - "RunAs": "DB-CP", - "Application": "DevSampleApp", - "Variables": [ - { - "DestDataFile": "DESTINATION_FILE" - }, - { - "SrcDataFile": "SOURCE_FILE" - } - ], - "When": { - "ToTime": "2100", - "FromTime": "0300" - } - }, - "Flow": { - "Type": "Flow", - "Sequence": [ - "GetData", - "UpdateRecords" - ] - } - }, - "JobsRunInDockerSample2": { - "Type": "Folder", - "ControlmServer": "IN01", - "RunAs": "controlm", - "SubApplication": "SampleSubApp", - "Application": "DevSampleApp", - "CommandJob": { - "Type": "Job:Command", - "SubApplication": "SampleSubApp", - "Host": "workbench", - "RunAs": "controlm", - "Application": "DevSampleApp", - "Command": "whoami ; pwd; ls -l" - } - } -} diff --git a/helix-control-m/102-build-docker-containers-for-automation-api/README.md b/helix-control-m/102-build-docker-containers-for-automation-api/README.md deleted file mode 100644 index e358eef..0000000 --- a/helix-control-m/102-build-docker-containers-for-automation-api/README.md +++ /dev/null @@ -1,23 +0,0 @@ -## Automation api container -How to build a container with configured Control-M Automation API CLI - -**AAPI_ENDPOINT** - Control-M endpoint host -**AAPI_TOKEN** - Control-M automation-api token - -```bash -SRC_DIR=. -AAPI_ENDPOINT= -AAPI_TOKEN= -sudo docker build --tag=controlm \ - --build-arg AAPI_ENDPOINT=$AAPI_ENDPOINT \ - --build-arg AAPI_TOKEN=$AAPI_TOKEN $SRC_DIR -``` - -Examples for running automation commands using the container -```bash -sudo docker run -it controlm ctm conf servers::get -sudo docker run -v $PWD:/src -it controlm ctm build /src/AutomationAPISampleFlow.json -``` - -See the [Automation API - Services](https://docs.bmc.com/docs/display/ctmSaaSAPI/Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/ctmSaaSAPI/Code+Reference) documentation for more information. \ No newline at end of file diff --git a/helix-control-m/102-build-docker-containers-for-automation-api/alpinelinux-ctm-cli/Dockerfile b/helix-control-m/102-build-docker-containers-for-automation-api/alpinelinux-ctm-cli/Dockerfile deleted file mode 100644 index 08886dd..0000000 --- a/helix-control-m/102-build-docker-containers-for-automation-api/alpinelinux-ctm-cli/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -#------------------------------------------- -# Container for Control-M Automation API cli -#------------------------------------------- - -FROM python:3.12-alpine -MAINTAINER hoshana Shandorfi - -ARG AAPI_ENDPOINT -ARG AAPI_TOKEN - -USER root -# alpine not include spec pycryptodome lib. -RUN pip install --no-cache-dir pycryptodome==3.21.0 - -# install ctm-automation-api kit -WORKDIR /root -RUN mkdir ctm-automation-api \ - && cd ctm-automation-api \ - && wget https://s3-us-west-2.amazonaws.com/controlm-appdev/release/latest/install_ctm_cli.py \ - && python install_ctm_cli.py \ - && ctm -v - - -# add controlm endpoint -RUN ctm env saas::add endpoint https://$AAPI_ENDPOINT/automation-api $USER $AAPI_TOKEN \ - && ctm env set endpoint diff --git a/helix-control-m/102-build-docker-containers-for-batch-application/JobsRunOnDockerSample.json b/helix-control-m/102-build-docker-containers-for-batch-application/JobsRunOnDockerSample.json deleted file mode 100644 index 6c54523..0000000 --- a/helix-control-m/102-build-docker-containers-for-batch-application/JobsRunOnDockerSample.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Defaults" : { - "Application" : "SampleApp", - "SubApplication" : "SampleSubApp", - "RunAs" : "controlm", - "Host" : "" - }, - - "JobsRunInDockerSample": { - "Type": "Folder", - "Comment" : "Code reviewed by John", - - "CommandJob": { - "Type": "Job:Command", - "Command": "whoami ; pwd; ls -l" - } - } -} diff --git a/helix-control-m/102-build-docker-containers-for-batch-application/README.md b/helix-control-m/102-build-docker-containers-for-batch-application/README.md deleted file mode 100644 index 207c74c..0000000 --- a/helix-control-m/102-build-docker-containers-for-batch-application/README.md +++ /dev/null @@ -1,24 +0,0 @@ -### Docker container for batch applications -For batch applications that should run in a container, use the following examples as a base container. - -Jobs in Control-M have a host attribute, either a host where the application and the Agent reside or a host group, which is a logical name for a collection of hosts. -``` -job1 { - "host" : "application_hostgroup" - "command" : "/home/user1/scripts/my_program.py" - "runAs": "user1" -} -``` -Specifying a job to run on a host group enables Control-M to balance the load by directing jobs to the various hosts in the host group. -Jobs will wait until a host is avalible in the group to start running. -Running the docker container with parameter CTM_HOSTGROUP=application_hostgroup self-registers the docker instance as a host and adds it to the host group. -Once a container instance is up and registered into Control-M, jobs waiting for "host"="application_hostgroup" will start running. - -#### centos7-agent -Example where an Agent is installed in the container. -When the container starts it registers the agent in Control-M -To remove the agent from the hostgroup, run decommission_controlm.sh script before stopping the container. - - -See the [Automation API - Services](https://docs.bmc.com/docs/display/public/workloadautomation/Control-M+Automation+API+-+Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/public/workloadautomation/Control-M+Automation+API+-+Code+Reference) documentation for more information. diff --git a/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/Dockerfile b/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/Dockerfile deleted file mode 100644 index d1cd5bf..0000000 --- a/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/Dockerfile +++ /dev/null @@ -1,74 +0,0 @@ -#------------------------------------ -# Control-M/Agent docker container -#------------------------------------ - -FROM almalinux:9.3-minimal-20231124 as base -MAINTAINER Shoshana Shandorfi - -ARG AAPI_ENDPOINT -ARG AAPI_TOKEN - -# install basic packages -RUN echo "Install Java, Python, and required utilities" \ - && microdnf update -y \ - && microdnf install -y --nodocs --noplugins --setopt=install_weak_deps=0 \ - util-linux psmisc which net-tools hostname binutils \ - jq \ - libnsl \ - ncurses \ - procps \ - shadow-utils \ - tar \ - tcsh \ - libxcrypt-compat \ - && microdnf install -y java-21-openjdk-headless --nodocs --noplugins --setopt=install_weak_deps=0 \ - && microdnf install -y python39 \ - && ln -s /usr/bin/python3.9 /usr/bin/python \ - && python3.9 -m ensurepip \ - && ln -s /usr/bin/pip3 /usr/bin/pip \ - && echo "Python version: " && python --version \ - && echo "Pip version: " && pip3 --version \ - && microdnf clean all - -# Add controlm user and modify sudoers file -ARG USERNAME=controlm -RUN echo "Create $USERNAME user" \ - && useradd -d /home/$USERNAME -s /bin/bash -g 0 -m $USERNAME \ - && chmod -R 775 /home/$USERNAME \ -# Make /etc/passwd writeable by group so missing arbitrary user entry can be added - && chmod g=u /etc/passwd - -# copy run and register controlm agent script to container -COPY run_register_controlm.sh /home/controlm/ -COPY decommission_controlm.sh /home/controlm/ - -# provision Control-M/Agent and copy without these build packages -FROM base as intermediate -ARG USERNAME=controlm - -# Install ctm-automation-api kit -RUN mkdir /root/ctm-automation-api \ - && cd /root/ctm-automation-api \ - && echo "Install Automation API CLI" \ - && curl -sO https://controlm-appdev.s3.us-west-2.amazonaws.com/deploy/9.22.10/install_ctm_cli.py \ - && python install_ctm_cli.py \ - && echo "ctm cli version: " \ - && ctm --version - - -USER controlm -WORKDIR /home/controlm - -ENV BMC_INST_JAVA_HOME=/etc/alternatives/jre_21_openjdk - -# Create AAPI env -# add controlm endpoint -RUN ctm env saas::add endpoint https://$AAPI_ENDPOINT/automation-api $AAPI_TOKEN \ - && ctm env set endpoint \ - && echo "Do I run" \ - && cd ~/ \ - && ctm provision image Agent_Alma.Linux \ -# enable controlm agent utilities - && echo "source /home/controlm/.bash_profile" >> /home/controlm/.bashrc \ - -CMD ["/home/controlm/run_register_controlm.sh"] \ No newline at end of file diff --git a/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/README.md b/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/README.md deleted file mode 100644 index 16dcf79..0000000 --- a/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/README.md +++ /dev/null @@ -1,38 +0,0 @@ -## How to build docker containers for Control-M - -Tutorial on the [product web page](https://docs.bmc.com/docs/display/ctmSaaSAPI/Building+a+docker+container+for+batch+applications) -that explains how to build a docker container for batch applications. - -To build container image of Control-M/Agent: -**AAPI_ENDPOINT** - Control-M Automation API endpoint -**AAPI_TOKEN** - Control-M user AAPI token - -```bash -SRC_DIR=. -AAPI_ENDPOINT= -AAPI_TOKEN= -sudo docker build --tag=controlm \ - --build-arg AAPI_ENDPOINT=$AAPI_ENDPOINT \ - --build-arg AAPI_TOKEN=$AAPI_TOKEN $SRC_DIR -``` - - -To run & self-register the containerize Control-M/Agent to Control-M: -**AGENT_TAG** - An agent tag name (use web interface or the CLI "ctm auth tokens::get -s forAgent=true") -**CTM_HOSTGROUP** - Application hostgroup -```bash -CTM_HOSTGROUP= -AGENT_TAG= -sudo docker run --net host \ - -e CTM_HOSTGROUP=$CTM_HOSTGROUP \ - -e AGENT_TAG=$AGENT_TAG -dt controlm -``` - -To decommission Control-M/Agent container and self-unregister from Control-M: -```bash -sudo docker exec -i -t /home/controlm/decommission_controlm.sh -sudo docker stop -``` - -See the [Automation API - Services](https://docs.bmc.com/docs/display/ctmSaaSAPI/Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/ctmSaaSAPI/Code+Reference) documentation for more information. \ No newline at end of file diff --git a/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/decommission_controlm.sh b/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/decommission_controlm.sh deleted file mode 100644 index 23fd900..0000000 --- a/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/decommission_controlm.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -CTM_ENV=endpoint -AGENT_NAME=$(hostname) - - -cd -#source .bash_profile - -echo remove agent [$AGENT_NAME] from hostgroup [$CTM_HOSTGROUP] -ctm config server:hostgroup:agent::delete IN01 $CTM_HOSTGROUP $AGENT_NAME -e $CTM_ENV -if [ $? -ne 0 ]; then - echo "Error deleting agent $AGENT_NAME from hostgroup $CTM_HOSTGROUP" - exit 1 -fi - -echo unregister controlm agent [$AGENT_NAME] from server IN01 -ctm config server:agent::delete IN01 $AGENT_NAME -e $CTM_ENV -if [ $? -ne 0 ]; then - echo "Error deleting agent $AGENT_NAME from the system" - exit 1 -fi - -exit 0 diff --git a/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/run_register_controlm.sh b/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/run_register_controlm.sh deleted file mode 100644 index c507f1d..0000000 --- a/helix-control-m/102-build-docker-containers-for-batch-application/centos7-agent/run_register_controlm.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -#use agent's java for provision setup -PATH=$PATH:~/bmcjava/bmcjava-V2/bin -CTM_ENV=endpoint -UNIQUE=$(head /dev/urandom | tr -dc A-Za-z | head -c 6 ; echo '') -AGENT_NAME=$(hostname)-$UNIQUE - -cd -pwd - -echo run and register controlm agent [$AGENT_NAME] with controlm IN01, environment [$CTM_ENV] -ctm provision saas:agent::setup $AGENT_TAG $AGENT_NAME -e $CTM_ENV -if [ $? -ne 0 ]; then - echo "Error registering agent $AGENT_NAME (tag:$AGENT_TAG) in Control-M/Server" - exit 1 -fi - -echo add or create a controlm hostgroup [$CTM_HOSTGROUP] with controlm agent [$AGENT_NAME] -ctm config server:hostgroup:agent::add IN01 $CTM_HOSTGROUP $AGENT_NAME -e $CTM_ENV -if [ $? -ne 0 ]; then - echo "Error adding agent $AGENT_NAME to agent host group $CTM_HOSTGROUP" - exit 1 -fi - -# loop forever -sleep infinity -exit 0 - \ No newline at end of file diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/README.MD b/helix-control-m/103-test-automation-JUnit-workflow-examples/README.MD deleted file mode 100644 index e04f3f3..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/README.MD +++ /dev/null @@ -1,89 +0,0 @@ -DISCLAIMER: This code was tested with version 9.20.120 - -## JUnit Control-M job using the automation api -How to verify control-m workflows runs using JUnit - - -**AUTOMATION_API_ENDPOINT** - Control-M endpoint -**API_KEY** - Control-M account token for automation - - -```java - - @Before - public void setUp() throws URISyntaxException { - Assert.assertFalse("you must update API_KEY and AUTOMATION_API_ENDPOINT in this class", "".equals(API_KEY)); - - endedOkJob = getJsonDefinitionFile("/JobEndedOk.json"); - endedNotOkJob = getJsonDefinitionFile("/JobEndedNotOk.json"); - twoMinJob = getJsonDefinitionFile("/TwoMinJob.json"); - threeJobFlow = getJsonDefinitionFile("/3JobsFlow.json"); - - conn = new Connection(AUTOMATION_API_ENDPOINT, API_KEY); - rs = new RunService(conn); - // NOTE: Do NOT set to false in production code. - rs.getApiClient().setVerifyingSsl(false); - } - - @Test - public void testIsJobFlowSuccedded() throws ApiException, TimeoutException{ - logger.info("testIsJobFlowSuccedded"); - rs.runJobs(new File("jobflow.json")); - boolean isEnded = rs.waitForJobToEnd("job1",15*1000).isJobStatus("Job1", JobStatus.ENDED_OK); - Assert.assertTrue(isEnded); - logger.info("end test testIsJobEndedUsingJobName"); - } - - -``` - -##### Executing this example -1. Generate java REST client for automation-api (see next section) -2. Change the API_KEY and AUTOMATION_API_ENDPOINT in the code to use your connection parameters -3. Change the example jobs (.json) in test/resources to match your environment Control-M Server and Agent's host (usually the same of the hostname where Control-M is installed) -4. Uncomment the correct twoMinJob definition, according to your agent's OS type - Windows or Linux /Unix in TestAutomationApiJUnitExamples class (in setUp method) -5. Use maven to build this project -Build project using maven from the home directory (where pom.xml reside) -```bash -mvn test -``` - - -##### Generate REST client -The tests uses a java client classes generated from the Automation Api [swagger.io](http://swagger.io) yaml specification -The url to the yaml sepecificion is \/yaml -```bash -wget /yaml -``` - -######Download the swagger code generator -For more information see [Swagger Code Generator](https://github.com/swagger-api/swagger-codegen) -```bash -wget https://repo1.maven.org/maven2/io/swagger/swagger-codegen-cli/2.2.1/swagger-codegen-cli-2.2.1.jar -O swagger-codegen-cli.jar -``` -######Generate the client -```bash -java -jar swagger-codegen-cli.jar generate -i -l java -o client/ -``` -* -i yaml location - file downloaded from previous step -* -l generated client programming language -* -o output location - - -if you have problem getting to the yaml (for example: uncertified SSL connection) consider copy the yaml to your current location - -######Build the client jar -Building the project using maven will add the jar to your local maven repository -```bash -cd client -mvn install -``` - -#####Execute the sample test -```bash -cd automation-api-quickstart/103-test-automation-JUnit-workflow-examples -mvn test -``` - -See the [Automation API - Services](https://docs.bmc.com/docs/display/public/workloadautomation/Control-M+Automation+API+-+Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/public/workloadautomation/Control-M+Automation+API+-+Code+Reference) documentation for more information. diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/pom.xml b/helix-control-m/103-test-automation-JUnit-workflow-examples/pom.xml deleted file mode 100644 index f9630ac..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/pom.xml +++ /dev/null @@ -1,62 +0,0 @@ - - 4.0.0 - bmc.control-m..examples - control-m.examples.testAutomationJunitExample - jar - Control-M testAutomationJunitExample - 9.19.0 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.5 - - 1.8 - 1.8 - - - - - - - - UTF-8 - 2.17.1 - - - - - - io.swagger - swagger-java-client - 1.0.0 - - - - org.apache.logging.log4j - log4j-api - ${log4j.version} - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - - - - junit - junit - 4.13.1 - test - - - diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/java/com/bmc/services/communication/Connection.java b/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/java/com/bmc/services/communication/Connection.java deleted file mode 100644 index a554127..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/java/com/bmc/services/communication/Connection.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.bmc.services.communication; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * This class is responsible for connecting to to the automation-api server - * @author gboham - * - */ -public class Connection { - private static final Logger logger = LoggerFactory.getLogger(Connection.class.getName()); - - private String apiClientBasePath; - private String apiKey; - - /** - * Connection model c'tor. - * @param apiClientBasePath ControlM Helix's AutomationApi endpoint - * @param apiKey apiKey token for authentication with ControlM Helix's AutomationApi - */ - public Connection(String apiClientBasePath, String apiKey) { - this.apiClientBasePath = apiClientBasePath; - this.apiKey = apiKey; - } - - /** - * ApiClientBasePath getter. - * @return AutomationApi endpoint - */ - public String getApiClientBasePath() { - return apiClientBasePath; - } - - /** - * apiKey getter. - * @return apiKey token - */ - public String getApiKey() { - return apiKey; - } -} diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/java/com/bmc/services/run/JobStatus.java b/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/java/com/bmc/services/run/JobStatus.java deleted file mode 100644 index 484bbc1..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/java/com/bmc/services/run/JobStatus.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.bmc.services.run; - -/** - * Enumeration representing job status - * (We don't want to use none type safe strings) - * @author ybergman - * - */ -public enum JobStatus { - ENDED_OK("Ended OK"), - ENDED_NOT_OK("Ended Not OK"), - WAIT_USER("Wait User"), - WAIT_RESOURCE("Wait Resource"), - WAIT_HOST("Wait Host"), - WAIT_WORKLOAD("Wait Workload"), - WAIT_CONDITION("Wait Condition"), - EXECUTING("Executing"), - STATUS_UNKNOWN("Status Unknown"); - - private String status; - - JobStatus(String sts){ - status = sts; - } - private String getStatusVal() { - return status; - } - - /** - * Get the correlated enumeration for the given string (or null) - * @param jobStatus - job status string - * @return - the right enumeration or null if the string is not a valid status string - */ - public static JobStatus toJobStatus(String jobStatus){ - for (JobStatus s : JobStatus.values() ){ - if (s.getStatusVal().equals(jobStatus)){ - return s; - } - } - return null; - } - - /** - * Job that ended are in one of the following statuses - */ - public static final JobStatus[] ENDED_STATUSES = {ENDED_OK, ENDED_NOT_OK, STATUS_UNKNOWN}; - - /** - * Job that failed to execute in one of the following statuses - */ - public static final JobStatus[] EXEC_FAILURE_STATUSES = {WAIT_USER, WAIT_HOST}; - -} diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/java/com/bmc/services/run/RunService.java b/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/java/com/bmc/services/run/RunService.java deleted file mode 100644 index 7bca340..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/java/com/bmc/services/run/RunService.java +++ /dev/null @@ -1,311 +0,0 @@ -package com.bmc.services.run; - -import java.io.File; -import java.io.InputStream; -import java.util.Arrays; -import java.util.concurrent.TimeoutException; - -import io.swagger.client.ApiClient; -import io.swagger.client.model.ActiveServices; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.bmc.services.communication.Connection; - -import io.swagger.client.ApiException; -import io.swagger.client.api.RunApi; -import io.swagger.client.model.JobRunStatus; -import io.swagger.client.model.JobStatusResult; -import io.swagger.client.model.RunResult; - -/** - * A service class that manage the RUN operations (cli: ctm run) such as: - * this.runJobs (json file) - order jobs - * this.waitForJobsToEnd (timeout) - wait for all the jobs in the current run to end - * this.waitForJobToEnd (specific job, timeout) - wait for specific job to end - * this.isJobStatus(specific job, status[]) - is job in certain status(es) - * this.confirm (specific job) - a wrapper for one of the many run operations - *

- * Usage example: - * Order file 3JobsFlow.json and wait for 15 seconds for a specific job (name "FirstJobOk") to ended successfully. - * Assert the result. - *

- * boolean endedOk = rs.runJobs(new File("3JobsFlow.json")).waitForJobToEnd("FirstJobOk", 15*1000).isJobStatus("FirstJobOk", JobStatus.ENDED_OK); - * Assert.assertTrue("job 3JobsFlow didn't end successfully as expected", endedOk); - *

- * (note that you can still get exceptions) - * - * @author ybergman - */ -public class RunService { - private static final Logger logger = LoggerFactory.getLogger(RunService.class.getName()); - - /** - * Constructor for using with {@link Connection} class - */ - public RunService(Connection conn) { - setApiKey(conn.getApiKey()); - setApiClientBasePath(conn.getApiClientBasePath()); - } - - private void setApiKey(String apiKey) { - api.getApiClient().setApiKey(apiKey); - } - - private void setApiClientBasePath(String serverURL) { - api.getApiClient().setBasePath(serverURL); - } - - public ApiClient getApiClient() { - return api.getApiClient(); - } - - /** - * Set client side certification for SSL. - * - * @param cert - */ - public void setSslCaCertification(InputStream cert) { - api.getApiClient().setSslCaCert(cert); - } - - /** - * internal interface to pass lambda expression as parameter - * - * @author ybergman - */ - interface IsEndedInterface { - public Boolean isEnded(String name) throws ApiException; - } - - private static final int DELAY_IN_SEC = Integer.parseInt(System.getProperty("DELAY_BETWEEN_JOB_STATUS_CHECK_IN_SEC", "2")); - - /** - * {@link Connection} connection to automation api server. - */ - Connection conn; - - /** - * {@link String} run id of the requested json job definition file (result after run) - */ - String runId; - - /** - * {@link RunApi} swagger generated client for Run api - * - * @see http://editor.swagger.io/#/ - */ - RunApi api = new RunApi(); - - public String getRunId() { - return runId; - } - - /** - * Order jobs using job definition file - * - * @param definitionsFile - json job definition file - * @return this instance - * @throws ApiException - internal error, check message for more info - */ - public RunService runJobs(File definitionsFile, File deployDescriptorFile) throws ApiException { - logger.debug("running job file {}", definitionsFile.getAbsolutePath()); - - RunResult res = api.runJobs(definitionsFile, deployDescriptorFile, null); - logger.debug("result of running job file {}: {}", definitionsFile.getAbsolutePath(), res); - runId = res.getRunId(); - // wait to make sure the job was ordered, otherwise we might ask from status before the job was processed (and executed) - sleep(5); - return this; - } - - public ActiveServices test() throws ApiException { - return api.getActiveServices(); - } - - /** - * overloading runJobs without deploy descriptor file - * - * @param definitionsFile - * @return this instance - * @throws ApiException - */ - public RunService runJobs(File definitionsFile) throws ApiException { - return runJobs(definitionsFile, null); - } - - /** - * Wait for the job(s) to end. The check is made every 2sec interval (may be change by of starting the jvm with -DDELAY_BETWEEN_JOB_STATUS_CHECK_IN_SEC=x) - * Job execution end status in one of the following statuses: Ended OK, Ended Not OK, Status Unknown - * - * @param timeout - timeout to wait in milliseconds - * @return this instance - * @throws ApiException - internal error, check message for more info - * @throws TimeoutException - an exception in case of the timeout occurred - * @see com.bmc.services.run#JobStatus - */ - private RunService waitToEnd(long timeout, IsEndedInterface isEndedInt, String field) throws ApiException, TimeoutException { - if (runId == null) throw new ApiException("there are no jobs to wait since runId is null"); - - logger.debug("waiting for run/job to end, timeout {}millisec", runId, timeout); - - long startTime = System.currentTimeMillis(); - Boolean ended = isEndedInt.isEnded(field); // can be change to isJobEnded(jobName) or areAllJobsEnded(), was done to allow 1 wait algorithm - while (!ended && System.currentTimeMillis() - startTime <= timeout) { - logger.debug("waiting for {}sec to check job(s) status again", DELAY_IN_SEC); - try { - Thread.sleep(DELAY_IN_SEC * 1000); - } catch (InterruptedException e) { - logger.debug("sleep (delay) was interrupted"); - } - ended = isEndedInt.isEnded(field); // <- can be change to isJobEnded(jobName) or areAllJobsEnded(), was done to allow 1 wait algorithm - } - if (!ended) { - logger.warn("timeout, waiting for job(s) to end exceed timeout {}", timeout); - throw new TimeoutException("waiting for jobs to end exceed timeout " + timeout); - } - - logger.debug("job(s) ended"); - return this; - } - - /** - * Wait for specific job in the current runId to end - * - * @param jobName - the job name - * @param timeout - timeout in milliseconds - * @return - this instance - * @throws ApiException - internal error, check message for more info - * @throws TimeoutException - an exception in case of the timeout occurred - */ - public RunService waitForJobToEnd(String jobName, long timeout) throws ApiException, TimeoutException { - RunService isEndedOper = waitToEnd(timeout, isEnded -> isJobEnded(jobName), jobName); - return isEndedOper; - } - - /** - * Wait for all the jobs in the current runId to end - * - * @param timeout - timeout in milliseconds - * @return - this instance - * @throws ApiException - internal error, check message for more info - * @throws TimeoutException - an exception in case of the timeout occurred - */ - public RunService waitForJobsToEnd(long timeout) throws ApiException, TimeoutException { - RunService isEndedOper = waitToEnd(timeout, isEnded -> areAllJobsEnded(null), null); - return isEndedOper; - } - - /** - * Dummy method to implement IsEndedInterface interface, act as a tunnel calling areAllJobsEnded() - * - * @param dummy - ignored - * @return - this instance - * @throws ApiException - internal error, check message for more info - */ - private Boolean areAllJobsEnded(String dummy) throws ApiException { - return areAllJobsEnded(); - } - - /** - * Check if all the jobs in the current runId ended - * - * @return this instance - * @throws ApiException - internal error, check message for more info - */ - public Boolean areAllJobsEnded() throws ApiException { - return areAllJobsInStatus(JobStatus.ENDED_STATUSES); - } - - /** - * Check if specific job in the current runId ended - * - * @param jobName - the job name (need to be one of the jobs in the current runId) - * @return this instance - * @throws ApiException - internal error, check message for more info - */ - public boolean isJobEnded(String jobName) throws ApiException { - JobRunStatus jobDetails = getJobDetailsByName(jobName); - if (Arrays.asList(JobStatus.EXEC_FAILURE_STATUSES).contains(JobStatus.toJobStatus(jobDetails.getStatus()))) { - throw new RuntimeException("job name " + jobName + " not executed due to configuration error"); - } - - if (jobDetails == null) throw new ApiException("job name " + jobName + " not found in runId " + runId); - - return Arrays.asList(JobStatus.ENDED_STATUSES).contains(JobStatus.toJobStatus(jobDetails.getStatus())); - } - - /** - * Get the job details of a job in the current runId - * - * @param jobName - job name - * @return - return - * @throws ApiException - */ - private JobRunStatus getJobDetailsByName(String jobName) throws ApiException { - JobStatusResult jList = api.getJobsStatus(runId, 0L); - for (JobRunStatus js : jList.getStatuses()) { - logger.trace("checking job {}, name {}", js.getJobId(), js.getName()); - if (js.getName().equals(jobName) && !"Folder".equals(js.getType())) { - return js; - } - } - return null; - } - - /** - * Check if all the jobs (in this.runId) ended - * - * @return this instance - * @throws ApiException - internal error, check message for more info - */ - public boolean areAllJobsEndedSuccessfully() throws ApiException { - return areAllJobsInStatus(JobStatus.ENDED_OK); - } - - public boolean areAllJobsInStatus(JobStatus... givenStatuses) throws ApiException { - if (runId == null) throw new ApiException("there are no jobs to check since runId is null"); - - JobStatusResult jobs = api.getJobsStatus(runId, 0L); - logger.debug("there are {} jobs to check", jobs.getTotal()); - - for (JobRunStatus currJob : jobs.getStatuses()) { - if (!Arrays.asList(givenStatuses).contains(JobStatus.toJobStatus(currJob.getStatus()))) { - logger.debug("job {} is not in one of the status:{} (status=" + currJob.getStatus() + ")", currJob.getName(), givenStatuses); - return false; - } - } - logger.debug("all {} jobs are in status {}", jobs.getTotal(), givenStatuses); - return true; - } - - public boolean isJobStatus(String jobName, JobStatus... statuses) throws ApiException { - if (runId == null) throw new ApiException("there are no jobs to check since runId is null"); - - JobRunStatus job = getJobDetailsByName(jobName); - logger.debug("job {} details: {}", job.getName(), job); - - return Arrays.asList(statuses).contains(JobStatus.toJobStatus(job.getStatus())); - } - - public void confirm(String jobName) throws ApiException { - if (runId == null) throw new ApiException("there are no jobs to confirm since runId is null"); - - JobRunStatus job = getJobDetailsByName(jobName); - if (job == null) { - logger.warn("job {} not found in runId {}", jobName, runId); - new ApiException("job " + jobName + " not found in runId " + runId); - } - logger.debug("job to confirm {} details: {}", job.getName(), job); - api.confirmJob(job.getJobId()); - } - - - private RunService sleep(int i) { - try { - Thread.sleep(i * 1000); - } catch (InterruptedException e) { - } - return this; - } -} diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/resources/log4j2.properties b/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/resources/log4j2.properties deleted file mode 100644 index d7ce0c9..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/main/resources/log4j2.properties +++ /dev/null @@ -1,35 +0,0 @@ -# Root logger option -name=PropertiesConfig -property.filepath = logs/tests.log -appenders = console, file - -# Direct log messages to stdout -appender.console.type = Console -appender.console.name = STDOUT -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n - -# Direct log messages to a log file -appender.file.type = File -appender.file.name = LOGFILE -appender.file.fileName=${filepath} -appender.file.layout.type=PatternLayout -appender.file.layout.pattern=[%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n -appender.file.append=false -appender.rolling.policies.type = Policies -# RollingFileAppender rotation policy -appender.rolling.policies.size.type = SizeBasedTriggeringPolicy -appender.rolling.policies.size.size = 10MB -appender.rolling.strategy.type = DefaultRolloverStrategy -appender.rolling.strategy.max = 10 - -# rootLogger -loggers=file -logger.file.name=com.bmc.services -logger.file.level = DEBUG -logger.file.appenderRefs = file -logger.file.appenderRef.file.ref = LOGFILE - -rootLogger.appenderRefs = stdout -rootLogger.appenderRef.stdout.ref = STDOUT -rootLogger.appenderRef.stdout.level= INFO \ No newline at end of file diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/java/com/bmc/services/communication/TestAutomationApiJUnitExamples.java b/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/java/com/bmc/services/communication/TestAutomationApiJUnitExamples.java deleted file mode 100644 index 8428692..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/java/com/bmc/services/communication/TestAutomationApiJUnitExamples.java +++ /dev/null @@ -1,180 +0,0 @@ -package com.bmc.services.communication; - -import java.io.File; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.concurrent.TimeoutException; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import com.bmc.services.run.JobStatus; -import com.bmc.services.run.RunService; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.swagger.client.ApiException; - - -public class TestAutomationApiJUnitExamples { - private static final Logger logger = LoggerFactory.getLogger(TestAutomationApiJUnitExamples.class.getName()); - - //TODO: change those parameter before build to hold your own configuration ! - private final static String AUTOMATION_API_ENDPOINT = ""; - private final static String API_KEY = ""; - - Connection conn; - File endedOkJob; - File endedNotOkJob; - File twoMinJob; - File threeJobFlow; - RunService rs; - - @Before - public void setUp() throws URISyntaxException { - Assert.assertFalse("you must update API_KEY and AUTOMATION_API_ENDPOINT in this class", "".equals(API_KEY)); - - //TODO - Uncomment the correct twoMinJob definition, according to your agent's OS type - Windows or Linux /Unix - //twoMinJob = getJsonDefinitionFile("/TwoMinJob_Windows.json"); - //twoMinJob = getJsonDefinitionFile("/TwoMinJob_Linux.json"); - Assert.assertFalse("uncomment relevant twoMinJob definition.", twoMinJob == null); - - endedOkJob = getJsonDefinitionFile("/JobEndedOk.json"); - endedNotOkJob = getJsonDefinitionFile("/JobEndedNotOk.json"); - threeJobFlow = getJsonDefinitionFile("/3JobsFlow.json"); - - conn = new Connection(AUTOMATION_API_ENDPOINT, API_KEY); - rs = new RunService(conn); - // NOTE: Do NOT set to false in production code. - rs.getApiClient().setVerifyingSsl(false); - } - - /** - * is specific job ended within 15 seconds - * - * @throws ApiException - * @throws TimeoutException - */ - @Test - public void testIsJobEndedUsingJobName() throws ApiException, TimeoutException { - logger.info("start test testIsJobEndedUsingJobName"); - rs.runJobs(endedNotOkJob); - boolean isEnded = rs.waitForJobToEnd("failJob", 15 * 1000).isJobEnded("failJob"); - Assert.assertTrue(isEnded); - logger.info("end test testIsJobEndedUsingJobName"); - } - - /** - * is specific job ended with a status "Ended Not Ok" within 15 seconds - * - * @throws ApiException - * @throws TimeoutException - */ - @Test - public void testIsJobStatuEndedNotOkUsingJobName() throws ApiException, TimeoutException { - logger.info("start test testIsJobStatuEndedNotOkUsingJobName"); - rs.runJobs(endedNotOkJob); - boolean isEndedWithStatus = rs.waitForJobToEnd("failJob", 15 * 1000).isJobStatus("failJob", JobStatus.ENDED_NOT_OK); - Assert.assertTrue(isEndedWithStatus); - logger.info("end test testIsJobStatuEndedNotOkUsingJobName"); - } - - /** - * wait for 20sec for job to end - if not, get a timeout exception - * - * @throws ApiException - */ - @Test - public void testJobStillExecuting() throws ApiException { - logger.info("start test testJobStillExecuting"); - boolean isJobEnded = rs.runJobs(twoMinJob).isJobEnded("twoMinJob"); - Assert.assertFalse(isJobEnded); - // wait 20sec for the job to end - timeout is thrown - try { - rs.waitForJobsToEnd(20); - throw new ApiException("should never get here, the above wait should throw timeout since the job takes 2min to end"); - } catch (TimeoutException e) { - } - - logger.info("end test testJobStillExecuting"); - } - - - /** - * wait 30sec for ALL jobs to end successfully - - * - * @throws ApiException - * @throws TimeoutException - */ - @Test - public void testAreAllJobsEndedSuccessfully() throws ApiException, TimeoutException { - logger.info("start test testAreAllJobsEndedSuccessfully"); - RunService jobNotOkRun = rs.runJobs(endedOkJob); - boolean isAllJobsEndedOk = jobNotOkRun.waitForJobsToEnd(30 * 1000).areAllJobsEndedSuccessfully(); - Assert.assertTrue(isAllJobsEndedOk); - logger.info("end test testAreAllJobsEndedSuccessfully"); - } - - /** - * check the invariants of more than 1 job - * 2 specific jobs ended ok, the last on should fail - * - * @throws TimeoutException - * @throws ApiException - */ - @Test - public void testJobFlow() throws ApiException, TimeoutException { - logger.info("start test testJobFlow"); - boolean jobRunResult; - // prepare your application's data - // run jobs - rs.runJobs(threeJobFlow); - // check if job 1 ended ok - jobRunResult = rs.waitForJobToEnd("FirstJobOk", 10 * 1000).isJobStatus("FirstJobOk", JobStatus.ENDED_OK); - Assert.assertTrue("job FirstJobOk failed", jobRunResult); - // check your application data - it is ok ? - // start job 2 (it was define with "confirm" so it waits for manual confirmation) - rs.confirm("SecondJobOk"); - // check if job 2 ended ok - jobRunResult = rs.waitForJobToEnd("SecondJobOk", 10 * 1000).isJobStatus("SecondJobOk", JobStatus.ENDED_OK); - Assert.assertTrue("job SecondJobOk failed", jobRunResult); - // check your application's data (the result of job 2) - // as a result of the data - job 3 should fail - make sure job 3 failed. - jobRunResult = rs.waitForJobToEnd("ThirdJobNotOk", 10 * 1000).isJobStatus("ThirdJobNotOk", JobStatus.ENDED_NOT_OK); - Assert.assertTrue("job ThirdJobNotOk didn't fail as expected", jobRunResult); - logger.info("end test testJobFlow"); - } - - /** - * wait for job to be in confirm status and confirm it - * - * @throws ApiException - * @throws InterruptedException - */ - @Test - public void testWaitForConfirmation() throws ApiException, TimeoutException, InterruptedException { - logger.info("start test testWaitForConfirmation"); - rs.runJobs(threeJobFlow); - int timeout = 15 * 1000; - long startTime = System.currentTimeMillis(); - - while (!rs.isJobStatus("SecondJobOk", JobStatus.WAIT_USER)) { - Thread.sleep(2000); - if (System.currentTimeMillis() - startTime > timeout) { - logger.error("SecondJobOk didn't get to user confirmation status for {}", timeout / 1000); - throw new TimeoutException("SecondJobOk didn't get to user confirmation status for " + timeout / 1000); - } - } - logger.debug("SecondJobOk is waiting for user confirmation"); - rs.confirm("SecondJobOk"); - boolean isEndedOk = rs.waitForJobToEnd("SecondJobOk", timeout).isJobStatus("SecondJobOk", JobStatus.ENDED_OK); - Assert.assertTrue("job SecondJobOk didn't end ok", isEndedOk); - logger.info("end test testWaitForConfirmation"); - } - - private File getJsonDefinitionFile(String file) throws URISyntaxException { - URL fileUrl = TestAutomationApiJUnitExamples.class.getResource(file); - return new File(fileUrl.toURI()); - } -} diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/3JobsFlow.json b/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/3JobsFlow.json deleted file mode 100644 index 6c1d830..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/3JobsFlow.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "Defaults": { - "Host": "HOST", - "RunAs": "USERNAME" - }, - "JobsFlowForJUnit": { - "Type": "Folder", - "OrderMethod": "Manual", - "FirstJobOk": { - "Type": "Job:Command", - "Command": "echo this is job 1 - ended successfully" - }, - "SecondJobOk": { - "Type": "Job:Command", - "Confirm": true, - "Command": "echo this is job 2 - ended successfully" - }, - "ThirdJobNotOk": { - "Type": "Job:Command", - "Command": "echo this is job 3 - ended with failure && exit 3" - }, - "flow1": { - "Type": "Flow", - "Sequence": [ - "FirstJobOk", - "SecondJobOk", - "ThirdJobNotOk" - ] - } - } -} \ No newline at end of file diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/JobEndedNotOk.json b/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/JobEndedNotOk.json deleted file mode 100644 index 43841e4..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/JobEndedNotOk.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "NotOk": { - "Type": "Folder", - "OrderMethod": "Manual", - "failJob": { - "Type": "Job:Command", - "Command": "invalid command, exit code none 0", - "Host": "HOST", - "RunAs": "USERNAME" - } - } -} \ No newline at end of file diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/JobEndedOk.json b/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/JobEndedOk.json deleted file mode 100644 index ba4ea06..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/JobEndedOk.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "Ok": { - "Type": "Folder", - "OrderMethod": "Manual", - "okJob": { - "Type": "Job:Command", - "Command": "echo job ended ok", - "Application": "OsApp", - "Host": "HOST", - "RunAs": "USERNAME" - } - } -} \ No newline at end of file diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/README.MD b/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/README.MD deleted file mode 100644 index a45b323..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/README.MD +++ /dev/null @@ -1,26 +0,0 @@ -## Changes to jobs definition files -Make sure you change the job definition files (in the directory) to include your Control-M components if needed (ControlMServer, Host, RunAs), -otherwise the jobs will fail or remain in wait state. - -Also, make sure you change AUTOMATION_API_ENDPOINT and API_KEY default values under TestAutomationApiJUnitExamples class. - -######JobEndedOk.json -```json - { - "Ok": { - "Type": "Folder", - "OrderMethod": "Manual", - "okJob": { - "Type": "Job:Command", - "Command": "echo job ended ok", - "RunAs": "controlm", <- agent's account - "Host": "agent_container", <- agent's name - "Application": "OsApp" - } - } - } -``` - - - - diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/TwoMinJob_Linux.json b/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/TwoMinJob_Linux.json deleted file mode 100644 index e2e2e2d..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/TwoMinJob_Linux.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "LongJob": { - "Type": "Folder", - "OrderMethod": "Manual", - "twoMinJob": { - "Type": "Job:Command", - "Command": "echo long waiting && sleep 2m && echo finish long waiting", - "Host": "HOST", - "RunAs": "USERNAME" - } - } -} \ No newline at end of file diff --git a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/TwoMinJob_Windows.json b/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/TwoMinJob_Windows.json deleted file mode 100644 index 5dbcd95..0000000 --- a/helix-control-m/103-test-automation-JUnit-workflow-examples/src/test/resources/TwoMinJob_Windows.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "LongJob": { - "Type": "Folder", - "OrderMethod": "Manual", - "twoMinJob": { - "Type": "Job:Command", - "Command": "echo long waiting && timeout 120 && echo finish long waiting", - "Host": "HOST", - "RunAs": "USERNAME" - } - } -} \ No newline at end of file diff --git a/helix-control-m/104-push-jobs-script-from-DEV-to-PROD/DeployDescriptorPROD.json b/helix-control-m/104-push-jobs-script-from-DEV-to-PROD/DeployDescriptorPROD.json deleted file mode 100644 index aef67f4..0000000 --- a/helix-control-m/104-push-jobs-script-from-DEV-to-PROD/DeployDescriptorPROD.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "DeployDescriptor": - [ - { - "Comment": "Assign Control-M Server", - "Property" :"ControlmServer", - "Assign" : "clm-aus-t2k2e4" - }, - { - "Comment": "Rename folder prefix from Dev to PROD", - "ApplyOn" : {"Type":"SimpleFolder"}, - "Property" : "@", - "Replace" : [ {"Dev_(.*)" : "PROD_$1"} ] - }, - { - "Comment": "Rename Job name prefix from Dev to PROD", - "ApplyOn" : {"Type":"Job"}, - "Property" : "@", - "Replace" : [ {"Dev_(.*)" : "PROD_$1"} ] - }, - { - "Comment": "Rename File Name from Dev to PROD", - "ApplyOn" : {"Type":"Job:Script"}, - "Property":"FileName", - "Replace" : [ {"Dev_(.*)" : "PROD_$1"} ] - }, - { - "Comment": "Assigns the RunAs property to the correct user for all jobs in the folder and subfolders.", - "Property": "RunAs", - "Assign": "ctmfinuser" - }, - { - "Comment": "Change File Path location of script", - "ApplyOn" : {"Type":"Job:Script"}, - "Property":"FilePath", - "Assign" : "C:\\Scripts" - }, - { - "Comment": "Change Host of a Job based on application Finance", - "ApplyOn" : {"Type":"Job"}, - "Property":"Host", - "Source":"Application", - "Replace" : [ { "Finance.*":"clm-aus-t2k2e4"} ] - }, - { - "Comment": "Change Host of a Job based on application Audit", - "ApplyOn" : {"Type":"Job"}, - "Property":"Host", - "Source":"Application", - "Replace" : [ { "Audit.*":"clm-aus-t2k2e4"} ] - }, - { - "Comment": "Change the In-Conditions", - "Property": "Events[0]['Event']", - "ApplyOn": {"Type": "WaitForEvents"}, - "Replace": [ {"Dev_(.*)-TO-Dev_(.*)": "PROD_$1-TO-PROD_$2"} ] - }, - { - "Comment": "Change the Out-Conditions", - "Property": "Events[0]['Event']", - "ApplyOn" : {"Type": "AddEvents"}, - "Replace": [ {"Dev_(.*)-TO-Dev_(.*)": "PROD_$1-TO-PROD_$2"} ] - } - ] -} \ No newline at end of file diff --git a/helix-control-m/104-push-jobs-script-from-DEV-to-PROD/README.md b/helix-control-m/104-push-jobs-script-from-DEV-to-PROD/README.md deleted file mode 100644 index 39c9619..0000000 --- a/helix-control-m/104-push-jobs-script-from-DEV-to-PROD/README.md +++ /dev/null @@ -1,29 +0,0 @@ - DISCLAIMER: This code was tested with version 9.0.20.080 - -## Pushing Jobs from DEV to PROD using Automation API - -This script can be used to push Jobs from Dev environment to a production environment according to the search criteria specified while changing specific job's properties -For a list of the property that will change see DeployDescriptorPROD.json. -This example will help you to get familiar with [deploy descriptor syntax](https://docs.bmc.com/docs/display/ctmSaaSAPI/Deploy+Descriptor). - - -## PROD Variables - -```bash -prodEndPoint=https:///automation-api -prodAapiToken= -deploydescriptor_path= -Temp_JobDef_path= -``` -## Dev Variables - -```bash -devEndPoint=https:///automation-api -devAapiToken= -``` - -For more detailed information please see script source file deployscript.sh - - -See the [Automation API - Services](https://docs.bmc.com/docs/display/ctmSaaSAPI/Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/ctmSaaSAPI/Code+Reference) documentation for more information. diff --git a/helix-control-m/104-push-jobs-script-from-DEV-to-PROD/deployscript.sh b/helix-control-m/104-push-jobs-script-from-DEV-to-PROD/deployscript.sh deleted file mode 100644 index e4d8a49..0000000 --- a/helix-control-m/104-push-jobs-script-from-DEV-to-PROD/deployscript.sh +++ /dev/null @@ -1,22 +0,0 @@ -#! /bin/bash - -# PROD -prodEndPoint=https:///automation-api -prodAapiToken= -deploydescriptor_path=/cygdrive/c/DeployDescriptorPROD.json -Temp_JobDef_path=/cygdrive/c/temp_job_file.json - -#Dev -devEndPoint=https:///automation-api -devAapiToken= - -# Download the job definitions and save on json# -tmp=$(curl -k -H "x-api-key: $devAapiToken" "Content-Type: application/json" "$devEndPoint/deploy/jobs?ctm=*&folder=Dev_Aud*") - -echo -e $tmp | sed 's/\\"/"/g;s/"{/{/;s/}"/}/' > /cygdrive/c/temp_job_file.json - -# dynamic job def to transform and deploy# -curl -k -H "x-api-key: $prodAapiToken" -X POST -F "definitionsFile=@$Temp_JobDef_path" -F "deployDescriptorFile=@$deploydescriptor_path" "$prodEndPoint/deploy" - -# Clean up temp file# -rm -f $Temp_JobDef_path \ No newline at end of file diff --git a/helix-control-m/201-automate-corrective-flow/AutomateCorrectiveFlow.json b/helix-control-m/201-automate-corrective-flow/AutomateCorrectiveFlow.json deleted file mode 100644 index 2118cd3..0000000 --- a/helix-control-m/201-automate-corrective-flow/AutomateCorrectiveFlow.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "SomeFlow": { - "Type": "Folder", - "SomeJob1": { - "Type": "Job:Command", - "Host": "agent", - "Command": "failme", - "RunAs": "controlm", - "DoCorrectiveFlowNeeded" : { - "Type": "If", - "CompletionStatus": "NOTOK", - "Correction": { - "Type": "Run", - "Folder": "CorrectiveFlow" - } - } - } - }, - - "CorrectiveFlow": { - "Type": "Folder", - "Comment": "Set Folder to run manual, so it will run only when needed", - "OrderMethod": "Manual", - "Host": "agent", - "CorrectiveJob1": { - "Type": "Job:Command", - "Command": "echo corrective", - "RunAs": "controlm" - }, - "CorrectiveJob2": { - "Type": "Job:Command", - "Command": "echo flow", - "RunAs": "controlm" - }, - "Flow2": { - "Type":"Flow", - "Sequence":["CorrectiveJob1","CorrectiveJob2"] - } - } -} diff --git a/helix-control-m/201-automate-corrective-flow/README.md b/helix-control-m/201-automate-corrective-flow/README.md deleted file mode 100644 index 638cf3b..0000000 --- a/helix-control-m/201-automate-corrective-flow/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Automatically run a different job to fix a problem resulting from the failure of the current job. - -When the current job fails this code triggers a different job to handle the error. -``` -"DoCorrectiveFlowNeeded" : { - "Type": "If", - "CompletionStatus": "NOTOK", - "Correction": { - "Type": "Run", - "Folder": "CorrectiveFlow" - } -} -``` -To run: -``` -ctm run AutomateCorrectiveFlow.json -``` - -See the [Automation API - Services](https://docs.bmc.com/docs/display/public/workloadautomation/Control-M+Automation+API+-+Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/public/workloadautomation/Control-M+Automation+API+-+Code+Reference) documentation for more information. diff --git a/helix-control-m/201-call-rest-api-using-curl/README.md b/helix-control-m/201-call-rest-api-using-curl/README.md deleted file mode 100644 index 431446e..0000000 --- a/helix-control-m/201-call-rest-api-using-curl/README.md +++ /dev/null @@ -1,27 +0,0 @@ -## How to invoke automation api REST calls using crul in a linux bash script -calling_automation_api_using_curl.sh contain example for differant api invocations such as Get Control-M Servers list and others - -```bash -#!/bin/bash - -endpoint= # HTTP end point in the format of https://:8443/automation-api -user= # Set this variable to Controlm-M user credentials -password= # Set this variable to Controlm-M user credentials -curl_params=-k # Out Of the Box the end point comes with self signed certificate, -k option accept such certificates - -# Session Login and get access API session token -login=$(curl $curl_params -H "Content-Type: application/json" -X POST -d "{\"username\":\"$user\",\"password\":\"$password\"}" "$endpoint/session/login" ) -echo $login - -token=$(echo ${login##*token\" : \"} | cut -d '"' -f 1) -echo token=$token - -curl $curl_params -H "Authorization: Bearer $token" "$endpoint/config/servers" # Get list of servers - - -# Session Logout to invalidate API session token -curl $curl_params -H "Authorization: Bearer $token" -X POST "$endpoint/session/logout" -``` - -See the [Automation API - Services](https://docs.bmc.com/docs/display/public/workloadautomation/Control-M+Automation+API+-+Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/public/workloadautomation/Control-M+Automation+API+-+Code+Reference) documentation for more information. diff --git a/helix-control-m/201-call-rest-api-using-curl/calling_automation_api_using_curl.sh b/helix-control-m/201-call-rest-api-using-curl/calling_automation_api_using_curl.sh deleted file mode 100644 index 2133bfd..0000000 --- a/helix-control-m/201-call-rest-api-using-curl/calling_automation_api_using_curl.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - - -endpoint= # HTTP end point in the format of https://:8443/automation-api -aapi_token= # Automation-Api token -curl_params=-k # Out Of the Box the end point comes with self signed certificate, -k option accept such certificates - -# using curl to GET information -ctm= # set this variable to Control-m a server name -orderno= # set this variable to orderno of a specific job -jobid=$ctm:$orderno -eventname= # set this variable to name of Event (Condition) to add/remove - -echo ctm=$ctm orderno=$orderno jobid=$jobid - -curl $curl_params -H "x-api-key: $aapi_token" "$endpoint/config/servers" # Get list of servers -curl $curl_params -H "x-api-key: $aapi_token" "$endpoint/config/server/$ctm/hostgroups" # Get list of hostgroups of a specific controlm -curl $curl_params -H "x-api-key: $aapi_token" "$endpoint/run/jobs/status?limit=4&jobname=*" # GET running jobs - -echo Retriving information on job with jobid=$jobid # GET running job output/status/log -curl $curl_params -H "x-api-key: $aapi_token" "$endpoint/run/job/$jobid/status" # GET running job status -curl $curl_params -H "x-api-key: $aapi_token" "$endpoint/run/job/$jobid/log" # GET running job log -curl $curl_params -H "x-api-key: $aapi_token" "$endpoint/run/job/$jobid/output" # GET running job output - -# using curl to Add/delete Event (Condition) -curl $curl_params -H "x-api-key: $aapi_token" -H "Content-Type: application/json" -X POST -d "{\"name\": \"$eventname\",\"date\":\"0505\"}" "$endpoint/run/event/$ctm" -curl $curl_params -H "x-api-key: $aapi_token" -X DELETE "$endpoint/run/event/$ctm/$eventname/0505" - -# using curl to build Jobs as code -curl $curl_params -H "x-api-key: $aapi_token" -X POST -F "definitionsFile=@../101-create-first-job-flow/AutomationAPISampleFlow.json" "$endpoint/build" - diff --git a/helix-control-m/201-call-rest-api-using-node-axios/node.js axios b/helix-control-m/201-call-rest-api-using-node-axios/node.js axios deleted file mode 100644 index f9fc79e..0000000 --- a/helix-control-m/201-call-rest-api-using-node-axios/node.js axios +++ /dev/null @@ -1,42 +0,0 @@ -//----------------------------------------------- -// NPM -//----------------------------------------------- -const axios = require('axios') -var https = require('https'); - -//----------------------------------------------- -// Global Variables -//----------------------------------------------- -const endpoint = 'https://:8443/automation-api'; -var aapi_token = 'your automation-api token'; - -//----------------------------------------------- -// Functions -//----------------------------------------------- -function RestCall(method, url, data) { axios({ - axios(endpoint .concat(url), { - method: method, - headers: {'x-api-key': `${aapi_token}` }, - httpsAgent: new https.Agent({ rejectUnauthorized: false }), - data: { data } - }).then(response => { - for ( var i = 0; i < response.data.statuses.length; i++) { - console.log(response.data.statuses[i].name, response.data.statuses[i].status); - } - return response - }) - .catch((error) => { - console.log(error) - }) - }) - .catch((error) => { - console.log(error); -}) - -} - -//----------------------------------------------- -// MAIN -//----------------------------------------------- -RestCall('get', '/run/jobs/status', '') - diff --git a/helix-control-m/201-call-rest-api-using-powershell/AutomationAPIExample.ps1 b/helix-control-m/201-call-rest-api-using-powershell/AutomationAPIExample.ps1 deleted file mode 100644 index 5a80300..0000000 --- a/helix-control-m/201-call-rest-api-using-powershell/AutomationAPIExample.ps1 +++ /dev/null @@ -1,114 +0,0 @@ -# Examples of invoking Control-M Automation API using Power Shell Invoke-RestMethod -# -# -# -#------------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------------ -# To accept self-signed certificates uncomment next line -# -#[System.Net.ServicePointManager]::ServerCertificateValidationCallback = {$true} - -#---------------------------------------------------------------- -$endPoint = "https:///automation-api" -$aapi_token= "Automation-Api token" -#------------------------------------------------------------------------------------ - - -$headers = @{ x-api-key="$aapi_token"} - -#---------------------------------------------------------------- -# Example of GET request -# -# Get Control-M Servers list -try -{ - $servers_res = Invoke-RestMethod -Method Get -Uri "$endPoint/config/servers" -Headers $headers - - "List of control-M servers:" - $servers_res -} -catch -{ - $error[0].ErrorDetails - $error[0].Exception.Response.StatusCode -} - -#---------------------------------------------------------------- -# Example of GET request -# -# Get Control-M Server agents from first server -$controlm=$servers_res[0].name -try -{ - $server_agents_res = Invoke-RestMethod -Method Get -Uri "$endPoint/config/server/$controlm/agents" -Headers $headers - - "List of Agents connected to control-M server '$controlm':" - $server_agents_res -} -catch -{ - $error[0].ErrorDetails - $error[0].Exception.Response.StatusCode -} - - -#---------------------------------------------------------------- -# Example of POST request -# -# Adding host to hostgroup in a Control-M Server -# -$hostgroup="mygroup1" -$agenthost="myhost1" - -$add_host_data = @{ host = "$agenthost"; } - -try -{ - "Adding host '$agenthost' to hostgroup '$hostgroup':" - - $uri="$endpoint/config/server/$controlm/hostgroup/$hostgroup/agent" - $add_to_hostgroup_res = Invoke-RestMethod -Method Post -Uri "$uri" -Body (ConvertTo-Json $add_host_data) -ContentType "application/json" -Headers $headers - - $add_to_hostgroup_res -} -catch -{ - $error[0].ErrorDetails - - $error[0].Exception.Response.StatusCode -} - -#---------------------------------------------------------------- -# Example of Uploading a file request -# -# Using the build to make sure that we have a valid source code -# -$uri="$endPoint/build" - -$FileName="AutomationAPISampleFlow.json" -$FilePath="c:\src\ctm-cli-v2\samples\$FileName" - -$fileBin = [System.IO.File]::ReadAllBytes($FilePath) -$enc = [System.Text.Encoding]::GetEncoding($CODEPAGE) -$fileEnc = $enc.GetString($fileBin) -$boundary = [System.Guid]::NewGuid().ToString() -$LF = "`r`n" -$bodyLines = ( - "--$boundary", - "Content-Disposition: form-data; name=`"definitionsFile`"; filename=`"$FileName`"", - "Content-Type: application/octet-stream$LF", - $fileEnc, - "--$boundary--$LF" - ) -join $LF -Try -{ - $res = Invoke-RestMethod -Uri $uri -Method Post -Body $bodyLines -ContentType "multipart/form-data; boundary=`"$boundary`"" -Headers $headers - - $res -} -Catch { - $error[0].ErrorDetails - - $error[0].Exception.Response.StatusCode -} diff --git a/helix-control-m/201-call-rest-api-using-powershell/README.md b/helix-control-m/201-call-rest-api-using-powershell/README.md deleted file mode 100644 index fbee842..0000000 --- a/helix-control-m/201-call-rest-api-using-powershell/README.md +++ /dev/null @@ -1,18 +0,0 @@ -## How to invoke automation api REST calls from PowerShell script -AutomationAPIExample.ps1 script contain example for differant api invocations such as Get Control-M Servers list and others -```PowerShell -# Get Control-M Servers list -try -{ - $servers_res = Invoke-RestMethod -Method Get -Uri "$endPoint/config/servers" -Headers $headers - - $servers_res -} -catch -{ - $error[0].ErrorDetails - $error[0].Exception.Response.StatusCode -} -``` -See the [Automation API - Services](https://docs.bmc.com/docs/display/ctmSaaSAPI/Services) documentation for more information. -See the [Automation API - Code Reference](https://docs.bmc.com/docs/display/ctmSaaSAPI/Code+Reference) documentation for more information. diff --git a/helix-control-m/201-call-rest-api-using-python/README.md b/helix-control-m/201-call-rest-api-using-python/README.md deleted file mode 100644 index b9b7fe5..0000000 --- a/helix-control-m/201-call-rest-api-using-python/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# How to make REST calls to Automation API in Python - -## Features: -- The user will be prompted for any required inputs if they are not specified on the command line -- disable SSL Certificate Signing (Useful when testing Automation API becuase it will deploy with a self signed cert by default) -- Enable verbose mode to see more details about the calls being made -- Gives a list of jobs in the AJF -- lets the users choose one to view the output -- After viewing output the user can enter 'q' to quit or chose another job - -## Usage: - usage: getoutput.py [-u USERNAME] [-p PASSWORD] [-h HOST] [-i] [-v] [--help] - - Connect to Control-M/Enterprise Manager via Automation API REST calls and display job outputs - - optional arguments: - -t Automation-Api token, --token token - Automation-Api token, used for identification and authorization - -h HOST, --host HOST Control-M/Enterprise Manager hostname - -i, --insecure Disable SSL Certification Verification - -v, --verbose Turn on verbose mode - --help show this help message and exit - -- HOST is the hostname where the Automation API Rest Server is running -- TOKEN is the Automation-Api token used to login to Control-M -- insecure will allow you to connect to an Automation API Rest Server that is using an untrusted certificate, this is useful because Automation API uses a self signed cert by default -- verbose will output the url that is being used eachtime as well as JSON header information -- help prints the above useage message - -## Rest in Python -Making rest calls in python is actually fairly simple using the requests package: [http://docs.python-requests.org/en/master/](http://docs.python-requests.org/en/master/) -### To do a get on a URL: -```python -r = requests.get(www.example.com/rest/end/point) # Makes r an object conaining information about the request and the response data -print(r.text) # r.text is the raw response -print(json.dumps(json.loads(r.text)['someObjects'][0])) # if the response if in JSON and you wish to output only certain feilds you can load it as a json object -``` -### To do a post on a URL: -```python -body = json.loads('{ "foo": "bar", "iAm": "postData" }') # use json.loads to make a json object to use as the post body -r = requests.post(www.example.com/rest/end/point/post, json=body) # json= automatically sets the content type for this request to json -if r.status_code == 200: # r.status_code holds the HTTP status code and can be useful for error handling in addition too the exceptions in the requests using a try except block - exit(0) -else: - exit(1) -``` -### Headers in a request: -```python -head = json.loads('{ "foo": "bar", "iAm": "headerData" }') # just like body content for a post make a json object -r = request.get(www.example.com/rest/end/point/requires-header-data, header=head) -``` \ No newline at end of file diff --git a/helix-control-m/201-call-rest-api-using-python/getoutput.py b/helix-control-m/201-call-rest-api-using-python/getoutput.py deleted file mode 100644 index b495f74..0000000 --- a/helix-control-m/201-call-rest-api-using-python/getoutput.py +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/python - -import collections -import json -import requests -import argparse -import sys -from getpass import getpass - -try: - from requests.packages.urllib3.exceptions import InsecureRequestWarning -except: - from urllib3.exceptions import InsecureRequestWarning - import urllib3 - -verbose = False -verify_certs = True - -def parse_inputs(): - parser = argparse.ArgumentParser( - description='Connect to Control-M/Enterprise Manager via Automation API REST calls and display job outputs', - add_help=False) - parser.add_argument('-t', '--token', dest='token', type=str, - help='Automation-Api token, used for identification and authorization') - parser.add_argument('-h', '--host', dest='host', type=str, help='Automation-API API endpoint host') - parser.add_argument('-i', '--insecure', dest='insecure', action='store_const', const=True, - help='Disable SSL Certification Verification') - parser.add_argument('-v', '--verbose', dest='verbose', action='store_const', const=True, - help='Turn on verbose mode') - parser.add_argument("--help", action="help", help="show this help message and exit") - - args = parser.parse_args() - - global verbose - global verify_certs - - verbose = args.verbose - insecure = args.insecure - - if insecure: # Use insecure to disable verifing SSL Cert on server useful becuase Automation API will use a selfsigned cert by default - if verbose: - print('Disabling SSL Cert verification') - try: - requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - except: - urllib3.disable_warnings(InsecureRequestWarning) - verify_certs = False - else: - verify_certs = True - - host = args.host - if host == None: - host = input("EM Hostname: ") - token = args.token - if token == None: - token = input("Automation-API token: ") - - baseurl = 'https://' + host + '/automation-api/' # Control-M Automation API endpoint url - login_args = collections.namedtuple('Login_Args', ['baseurl', 'token']) - auth = login_args(baseurl, token) - return auth # return auth info as named pairs - -def list_jobs(baseurl, token): - - global verbose - jobstatusurl = baseurl + 'run/jobs/status' # url to list statuses of all the jobs in the AJF - - data = json.loads('{"x-api-key":"' + token + '"}') # the jobs statues call should have the token in the header as JSON - - if verbose: - print('Job Status URL: ' + jobstatusurl) - print('Job Status Header: ' + json.dumps(data)) - - r2 = requests.get(jobstatusurl, headers=data, - verify=verify_certs) # do a get on the job status url returns json with all of the job status - - if 'statuses' in json.loads(r2.text): # if statuses exsits in json response store the statuses to variable statuses - statuses = json.loads(r2.text)['statuses'] - else: - print('No job statuses were loaded.') # if statuses does not exist, report it. this can happen if no jobs are in the AJF - return(2) - if verbose: - print('statuses:\n' + json.dumps(statuses)) - - length = len(json.loads(r2.text)['statuses']) # check how many jobs are in statuses - - if verbose: - print('length: ' + str(length)) - while True: - x = 0 - while x < length: # iterate though statuses - # contents of python json objects can be accessed like named tupel for key value pairs and arrays for repeated objects - # the end result is similar to jsonpath expressions statuses[3]['jobId'] is equivalent to 3.jobId, you can test and learn about jsonpath expressions here: https://jsonpath.curiousconcept.com/ - print( - str(x) + '. ' + statuses[x]['jobId'] + ', ' + statuses[x]['name'] + ', ' + statuses[x]['status'] + ', ' + - statuses[x]['type']) - x += 1 - - selected = input("Enter a number to see it's output, or q to quit: ") - - if selected == 'q': - return(0) - try: - selected = int(selected) - outputurl = statuses[selected]['outputURI'] # get the outputURI from the select job in statuses - print_output(outputurl, data) - except TypeError: - print("Please enter either a number or q to quit") - except ValueError: - print("Please enter either a number or q to quit") - except IndexError: - print('That does not exist') - -def list_servers(baseurl, token): - - global verbose - getAgentsUrl = baseurl + 'config/servers' # url to the servers list - - data = json.loads('{"x-api-key":"' + token + '"}') # the jobs statues call should have the token in the header as JSON - - if verbose: - print('Servers URL: ' + getAgentsUrl) - print('Servers Header: ' + json.dumps(data)) - - r2 = requests.get(getAgentsUrl, headers=data, - verify=verify_certs) - if verbose: - print "Response:" + json.dumps(r2.text) - - if len(json.loads(r2.text)) > 0: - servers = json.loads(r2.text) - else: - print('No servers are connected to the system.') - return(2) - - if verbose: - print(str(len(servers)) + " servers exist in the system") - x = 0 - while x < len(servers): - print( - str(x+1) + '. Name:' + servers[x]['name'] + ', Host:' + servers[x]['host'] + ', Status:' + servers[x]['state']) - x += 1 - return 0 - -def print_output(outputurl, data): - global verbose - if verbose: - print(outputurl) - - r3 = requests.get(outputurl, headers=data, - verify=verify_certs) # go a get on the outputURI, outputURI already includes the current token - print(r3.text) # this request returns the raw text output of the job and not json - - -args = parse_inputs() -print args -print "----------- list servers ------------" -list_servers(args.baseurl, args.token) -print "----------- jobs status ------------" -list_jobs(args.baseurl, args.token) diff --git a/helix-control-m/201-call-rest-api-using-python/requirements.txt b/helix-control-m/201-call-rest-api-using-python/requirements.txt deleted file mode 100644 index 5675524..0000000 --- a/helix-control-m/201-call-rest-api-using-python/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -urllib3 -requests \ No newline at end of file diff --git a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/AI Kubernetes.ctmai b/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/AI Kubernetes.ctmai deleted file mode 100644 index 7742626..0000000 --- a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/AI Kubernetes.ctmai +++ /dev/null @@ -1 +0,0 @@ -H4sIAAAAAAAAAO0b2W7buPa9X0HoaS7QZLQvHRS4bux2PHXswE47GDRBQMt0rRtZVLU4CQz/+1C0dpGy3KSTuUH1YotnI89Omt6+AkCAvi+8AQBsyQt59eAakXfh46d3g+l4cDmYCa/3kIUT+i58GKcIvSH4GM9R4KEIhTkOWsLYjS4ffIpzNhoWkNBOhm7NEPwPz4Hvxl8dD8Sh430Ft4SRHbkZrgvD6BwvnKWDFgnNGG/QmogCsvwayKIsZYgbFIQO9hIc81Q8FcVTWRQLib6LH9bIiz7naIqZAu/ChftpOgrJ2JfrdCxAYVQfC521G8OIUP+OwygRlLEPYi9y1ugCBnBdIfEbI+ge2XHCI9ELhYBc4XVwpt5Uk6k0goX9kNopeb6kg8WzLY0IUap+P0Ane+Yo50MRnHDgwblLtRsFMaoA7fUin2OTeYl9Yd0ybWb7AREslKC76/xl97rb/L977sXT1BR7UYT2DHsLJzHDLEJ+TQMsgsQ7CGZONvGj1Bc30I3RZEldQ2iutUr2OUFOqEggug8H0fdME3SbijuEP/FRACMcUH1+E2rYu+s6fYt1GfZtIKygt3DR1GYYh8KD2EXl2CiBsszTY+ohC4g/8HwWwShOmAhiEw3dRwFRzdBb4k4mjB2aYkjyMEVdNmTZsmRN1AyWYolrILuuVLLgVlzmYilC4TA29iLoMI25wCU+dWhzPeUVmbqmyKqsSLKkaabK4E75F9NIdcdGJMDfqXH7MIKVGGubzZ6wEEGSYzRZjhwPMaWAMgaxcxBd4NDJiOEyQsHMRzapDPYl4XuYx8BblDnM0RIHqBOLAG0cdEdnSghJyTqdw8he/eo7wA4QjBDL7JQ0RDCwV+9xkPCni6CiKXUnGjJp6hXtYoL15p0LvduQE2uF+hOjcq3GsxshvUXIH+zJmem2gUyTU1bBPpqzPybvxr3zQT3r7J8dY7Q51sxQpdghubOsqA7J7Z7aYrtNO44bF9u7HaCpF5wswXb7ANfubtfMK5WkSsUy8xvhhLzFFH2LSTNxiSdx5MeJyCV0Qz526GMvRB3RM7T3jkvCoZyKQuprncjyqlNVGqMyZ0+jXpZqwTn2HDKHM7z2XcQsS3l2J72Ys3wAmE6bYAK7hai1Fj2i1HxXLak+/MaCpzDKsig3oqHKsqxLpqSKsmqpnPx8RM3pUHU6151DlYeb7fMFKqZkKpolqZJqmLqhKLw8Vq5AIYomt6x0wUoDzUQggVls2wgtmsmg1bfb1lSsSFc0Q9IURbd0UdL1Z7AXO8Ap2iOtpauGpJuyaMiWQT7kLsZaQsfl4R3uFlrqzuGSxScG+3lNgtQTDkyUEuCSqUi55yBy/afS4Rw921v0kMj9c4VdBFx+Y0SRN1navkvQaXPCm233mtsxtN4TLTLjqjbSueomRwKBM0fJeUD463ZbNAysAvz/V1kb6I5H4Ju9PuVa4S29/YMbZf4mP6vXH1AESJ0EmC4RkJgGZy6CXuzX9/5HHAvUwK2VnFvHO1Xx+n6wziN1TWSvMLgSTo56roQrLyV8Pxz3RuBi0gf9wexsOry4HE7GJfjxjI8JlkeISQkJKzCafJg96ZRd/DX8MdOdDmaTT9OzAdH2aHA5eGJFk26Uo2Yi5t6JQMPJ2o+OGp0z59iIbjlxHO6ZtO9A8iwTshueelI/Jn3ykycjOr8ncx6VNzk7r2c6WTycMPvUf15ScnSWZNlXQrY3vhLAW/JKPq5/A9EKkagAnCDabgMU4jiw0bBP4574GuCjlzbgV97S+TFR1no4ywmyBcumx57Mdj+X7Ry83Hjs1vj8DN4MIQveM3qS9JKCtx5q+7OylrOuZwwzm6X9n2H28sKsnzazLznQ8oa9WgT/PdG2YBvhZ7w9U7z5OIz+nT/Xp99y8xVXHPbPgesIcEHUegGjFctcAvI2n2GQ/WhUXTi6t92Y7G3OVjAIm+CvAY59esCHg3VdZUNvUKNuynbhHNHTGGJLb398Ci4CvHRc9KbKL28QJuObi+mkCsw0HJFMMcf3VWB6xArEU1GXVVm0DMXQRF2zNLOGGKIB1QVzqiTCnQX1R6oI6Lr1Bffj9fqh7omHnC9XoV0L7QOLap1rm0nbnaGb2Q7p4oDb5BcNhjfTT+PeTGD7xDT2QC98w7SmeCrriipZlqbKpqnKUvW+gECciOSbKL8slbyXTvy6poSQ7Dtqv8fns2ek42LuPQYQ02rx3ddjDlzP2FSvsVQTP/9EmyuGuY0uy0n3ZI8WxGz5y4LSrvQJVsSrt5tGSa4KK71dd02ALA+UOL6smKqlJ1dfFEMVdbXmy2mPRONt6C3QPSESuRiFC1RQQpTUDFJ/i55l5kMbAZvEaHK/IPgl/A+HhKbP44NnHwzM2GGA8tB5+/YtGCQH9eCv3vkI4ABM0/YNDPuAQHlR9eVRtjE4tjEkQ9aIZWRDtSTZsrrroWUCrbn9H83ByRaUY4i/CIi0YYibgFXFsCTTUk3D0iVVP6xj5al9sjUKntRjiUS2libTH+OP9eYkV7tEOhhD1SRLt8iHYr4whyz2ary2gIOQa8jSNFnVTZm0B4pGfPSwrtVny5Wd7goWd8KLG11V6VQhyQaWe26Q237/O+JhnWg1ESt8N/SStmaDMvP/bHI7+HPpUILj0B/3GGCEbdjsGkuBr1mWJCmybMqipCnWYRvqJUfM9pD0MzsJEDx01/MWYxzN4IZub0t6EBx77/znKLCT8x0nRDdTFBWXOZJG40PgZyLPYRSBESR786CKMFgnNARjTTBW6O7GpUj/na/tUxtnqUiI07VasqnN57ZyYmsL9US1NenE1CTzxLBJ2psbBjK0eS4g88hiX175ZwBpUYO+Y6dNd7XdLnw0t1YzvAQbhigcx8n/H5LGKwXkLr+tIUi1YGAIYYZxnU0mh35m5tqggPt3C+h5OIKlWzD5/xsKCDE0ncGs97mQXYLvW+Qi6yVor7Jp0IUJf876oxkBbnevdn8DgqKEE8AyAAA= \ No newline at end of file diff --git a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/README.MD b/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/README.MD deleted file mode 100644 index ad1c017..0000000 --- a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/README.MD +++ /dev/null @@ -1,268 +0,0 @@ -# Best Practices for Using Helix Control-M to Run a Pod to Completion in a Kubernetes-based Cluster - -## Running Control-M Workloads in Kubernetes - -This readme describes how to run Helix Control-M workloads in Kubernetes-based clusters, including OpenShift (from -RedHat) and public cloud platforms such as EKS (from Amazon), AKS (from Microsoft Azure), or GKE (from Google). The -information in this document focuses on the scenario of running a pod to completion with a Helix Control-M/Agent running -within the cluster. - -The document demonstrates how to do this while leveraging Helix Control-M scheduling and monitoring capabilities. - -> The methodology and configuration described in this document were tested and verified by BMC. The recommended best -> practices do not impact or change BMC licensing considerations. - -## Working Assumption - -The guidelines and configuration in this document are based on the Amazon Elastic Kubernetes Service and were tested in -a Linux environment with the September 2022 release of Helix Control-M. -Testing was done on an Amazon Elastic Kubernetes Service (Amazon EKS) cluster of version 1.20. If you have an Amazon EKS -cluster of version 1.23, you must install the Amazon EBS CSI driver. For more information, -see [Amazon EBS CSI driver](https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html). - -## Methodological Approach - -There are several ways to run a Kubernetes (K8s) pod to completion using Helix Control-M. - -This document describes a selected methodology for utilizing a Helix Control-M Application Integrator job type to run -your Helix Control-M workloads in Kubernetes. - -The Control-M Application Integrator job starts a Kubernetes job entity and monitors its status until the job ends. - -After the job ends, the pod’s output is captured and is presented in the Helix Control-M job output. The Kubernetes job -is then deleted, so that another job with the same name can be run. - -The Agent pod is run as a StatefulSet, so that the hostname will be identified consistently across pod shutdown and -startup. This enables Helix Control-M to uniquely identify the Helix Control-M/Agent continuously and consistently. - -The pod uses a persistent volume, so that job data and Helix Control-M/Agent state are kept during shutdown and startup. - -## Sample Files - -> The following files in this directory are sample files. You will need to edit these samples and customize them to -> match the unique needs of your environment. - -| File Name | Description | -|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| build_docker_example.sh | A bash script that is used to build a Docker image | -| Dockerfile | A file that contains instructions for how to construct a Helix Control-M/Agent Docker image | -| startup.sh | A bash script that is used to start up the Helix Control-M/Agent | -| ctmhost_keepalive.sh | A bash script that is used to keep the deployed pod alive | -| AI Kubernetes.ctmai | A Kubernetes job type definition file | -| stateful_ha.yaml | A file that contains StatefulSet definitions for running a Helix Control-M/Agent container, Kubernetes definitions for creation of the Persistent Volume Claim, and a linkage between them | -| sleep_for_20sec_job.yaml | A sample Kubernetes job for testing: Print the current time and sleep for 20 seconds | -| roles.yaml | A sample file used to enforce RBAC authorization in the test environment | -| testAiConnectionProfile.json | A json configuration file for creating a connection profile that uses the kubectl utility in the Agent container | -| testAIJob.json | A json configuration file for creating the Control-M Application Integrator KUBERNETES job type | - -## Required Control-M Authorizations - -To perform the processes described in this document, you must have various Control-M authorizations configured: - -- For building and executing the Helix Control-M/Agent container and deploying the Application Integrator job type: - - - Provisioning of agents and applications, as described - in [API Provision service authorizations](https://documents.bmc.com/supportu/controlm-saas/en-US/Documentation/Automation_API_Authorizations.htm#APIProvisionServiceAuthorizations) - - - Access to Control-M Application Integrator, as described - in [API Build and Deploy service authorizations](https://documents.bmc.com/supportu/controlm-saas/en-US/Documentation/Automation_API_Authorizations.htm#APIBuildandDeployServicesAuthorizations) - -- For creating and monitoring jobs that run the Kubernetes pod: - - - Folder authorizations, as described - in [Role Authorizations](https://documents.bmc.com/supportu/controlm-saas/en-US/Documentation/Users_and_Roles.htm#RoleAuthorizations) - - - Job permissions, as described - in [Role Authorizations](https://documents.bmc.com/supportu/controlm-saas/en-US/Documentation/Users_and_Roles.htm#RoleAuthorizations) - -## Process Steps - -A typical process of deploying Helix Control-M/Agent on Kubernetes consists of the following steps: - -- [Step 1: Create a Helix Control-M/Agent image](#step-1-create-a-helix-control-magent-image) - -- [Step 2: Add Kubernetes job type to Application Integrator](#step-2-add-kubernetes-job-type-to-application-integrator) - -- [Step 3: Create persistent storage for the Agent and run the Agent](#step-3-create-persistent-storage-and-run-the-agent) - -- [Step 4: Verify permissions](#step-4-verify-permissions) - -- [Step 5: Run Helix Control-M jobs](#step-5-run-control-m-jobs) - -> The processes described here are meant as guidelines for running workloads in Kubernetes-based clusters. These -> processes have been tested in BMC environments, but may need to be adjusted to run in your environments. If you need -> support in deploying Helix Control-M in this manner, please request assistance from BMC Support. - -### Step 1: Create a Helix Control-M/Agent image - -Edit the build_docker_example.sh script with local values for the variables that appear in uppercase, and then run the -script. - -The script contains the following variables: - -- AAPI_END_POINT - An endPoint is the URI for Control-M Automation API. The endpoint has the following format: - https://-aapi..controlm.com/automation-api - -- AAPI_TOKEN - The token value for connecting to the Automation API server. For information about obtaining this token, - see [Creating an API Token](https://documents.bmc.com/supportu/controlm-saas/en-US/Documentation/Creating_an_API_Token.htm). - -- AAPI_CLI_URL - The full URL path of the .BIN installer file for Automation API CLI (for example, - DR5V3.9.0.20.220_Linux-x86_64.BIN), as it appears in the Plugins section in Helix Control-M. - -- AGENT_IMAGE_NAME - The image name of the Agent. For more details, - see [Provision Service](https://docs.bmc.com/docs/display/ctmSaaSAPI/Provision+service). - -This produces an image that contains a Helix Control-M/Agent with kubectl and support for executing Control-M -Application Integrator job types. - -The container contains the following items: - -- Control-M/Agent (including Control-M Application Integrator and Control-M Automation API CLI) -- Control-M Application Integrator KUBERNETES plugin -- Java 11 run time -- Python 3.9 -- kubectl -- Bash startup scripts - -Automation API instructions in the Dockerfile are invoked during the build of the image for provisioning of an Agent. - -When the container is loaded, the startup.sh is invoked. The startup script performs the following actions: - -1. Mounts the container to use the persistent volume. - -2. Registers and configures the Agent in Helix Control-M. - -3. Adds the Agent to a hostgroup. This ensures high availability and load balancing between the Agents. - > Note that if you want to add multiple Agents, you can change the value of the **replicas** setting in the * - *stateful_ha.yaml** file. - -4. Deploys the Control-M Application Integrator KUBERNETES job type to enable running it with the Agent. - -5. Executes the ctmhost_keepalive.sh script. This script verifies the Agent-Server connection and writes its output to - the pod’s standard output. - -### Step 2: Add Kubernetes job type to Application Integrator - -Use Control-M Application Integrator to import the AI Kubernetes.ctmai file: - -1. In Helix Control-M, under the **Tools** menu, click **Application Integrator** to open Control-M Application - Integrator. - -2. Click **Import job type from file**, and choose the **AI Kubernetes.ctmai** file that you downloaded earlier. - -3. Click *Deploy* and select one of the existing agents. - -This creates a new job type named KUBERNETES. This job type can now be used on any Agent associated with Helix -Control-M. - -### Step 3: Create persistent storage and run the Agent - -Use the stateful_ha.yaml file to allocate persistent storage for Agent usage and run the Agent, so that the state of the -Helix Control-M/Agent is kept across shutdown and startup of the pod. - -Before applying the file, perform the following steps: - -1. Edit the following parameters in the stateful_ha.yaml file: - - - AAPI_END_POINT - An endPoint is the URI for Control-M Automation API. The endpoint has the following format: - https://-aapi.controlm.com/automation-api - - - AGENT_TOKEN_TAG - The token name of the agent, as it appears in Agent Token Management in Helix Control-M. - - - AGENT_HOSTGROUP_NAME - The name of a host group that the deployed Agent will be added to. - - - PERSISTENT_VOL - The name of the directory of the persistent volume. - - - DOCKER_IMAGE_NAME - The image that will be deployed in the container. - -2. Prepare an API token for connecting to the Automation API server: - - a. Create an API token as described - in [Creating an API Token](https://documents.bmc.com/supportu/controlm-saas/en-US/Documentation/Creating_an_API_Token.htm). - - b. Run the following command to create a secret for the API token that is referenced by the AAPI_TOKEN parameter in - the .yaml file: - - ``` - kubectl create secret generic credentials --from-literal=AAPI_TOKEN= - ``` - -Run the following command from the directory that contains the yaml file that controls the cluster: - -``` -kubectl apply -f stateful_ha.yaml -``` - -After executing the apply command, verify that the pod is running through the pod logs. After all setup processes of the -Agent have completed (in the startup.sh script), the ctmhost_keepalive.sh script runs in an endless loop. - -After the pod is loaded successfully, the pod’s output log should contain a report regarding the connection between the -Helix Control-M/Agent to Helix Control-M. - -To verify the connection between the Helix Control-M/Agent and Helix Control-M, run the following checks: - -| Check | Command | Expected Output | -|----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------| -| Helix Control-M/Agent side | ```kubectl exec -it ctm-statefulset-agent-0 -- bash -c "source .bash_profile && ag_ping"``` This command runs the ag_ping utility, which resides in the Helix Control-M/Agent image. The StatefulSet is named satefulset-agent, with a number suffix for the instance number (0). | Server is alive. Result: Success. | -| Helix Control-M side | Access Helix Control-M, navigate to the Configuration domain, and review the list of available agents. | Agent : ctm-statefulset-agent-0 is alive | - -> By default the PersistentVolumeClaim accessModes property is configured to ReadWriteOnce. You can use ReadWriteMany -> value when needed. - -### Step 4: Verify permissions - -Verify with your Kubernetes Administrator that the Agent's pod has permissions to start the requested "run pod to -completion". - -> In the BMC test environment, the **roles.yaml** file was applied, to ensure that the Agent can apply Kubernetes jobs -> from inside the pod. -> To attach this role for StatefulSet, needs to specify **serviceAccountName: node-apis** under **containers** section - -### Step 5: Run Helix Control-M jobs - -To run the Helix Control-M job, perform the following actions: - -1. Create a connection profile between the deployed Agent and KUBERNETES job type. In the connection profile you need to - supply the kubectl path in the Agent container. This connection profile synchronizes between the kubectl utility in - the Agent and the KUBERNETES job type. - - For example, execute the following API command for the one-time creation of the connection profile: - ``` - ctm deploy testAiConnectionProfile.json - ``` - -2. Create a new KUBERNETES type job and point it to the created Agent or Agent host group. Supply the created connection - profile and a yaml file path or Kubernetes resource id (object UID) and run the job. - - For example execute the following API command to run the Control-M Application Integrator KUBERNETES job: - - ``` - - ctm run testAIJob.json - - ``` - -> Before running the job, ensure that the provide sleep_for_20sec_job.yaml file is available in the container under -> /home/controlm - -3. Check job status and output. - - To check the job status run: - - ``` - - ctm run status - - ``` - - > run id is obtained from the output of the run command. - - To check the job output run: - - ``` - - ctm run job:output::get - - ``` - - > job id is obtained from the output of the run status command. \ No newline at end of file diff --git a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/build_docker_example.sh b/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/build_docker_example.sh deleted file mode 100644 index 47e6189..0000000 --- a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/build_docker_example.sh +++ /dev/null @@ -1,9 +0,0 @@ -# This is a build command example with parameters -# You need to set : -# 1. AGENT_IMAGE_NAME - the agent image you want to install (taken from "ctm provision images Linux" cli) -# 2. AAPI_END_POINT - An endPoint is the URI for Control-M Automation API. The endpoint has the following format: https://-aapi..controlm.com/automation-api -# 3. AAPI_TOKEN - The token value for connecting to the Automation API server. For information about obtaining this token, see [Creating an API Token](https://documents.bmc.com/supportu/controlm-saas/en-US/Documentation/Creating_an_API_Token.htm) -# 4. DOCKER_IMAGE_NAME - the docker image name and optionally a tag in the 'name:tag' format -sudo docker build --build-arg AGENT_IMAGE_NAME=Agent_CentOS --build-arg AAPI_END_POINT="$AAPI_END_POINT" --build-arg AAPI_TOKEN="$AAPI_TOKEN" -t "$DOCKER_IMAGE_NAME" docker - -# Don't forget to upload the result to your docker repository (ECR, DockerHub, etc.) for k8s use. \ No newline at end of file diff --git a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/docker/Dockerfile b/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/docker/Dockerfile deleted file mode 100644 index 3d23367..0000000 --- a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/docker/Dockerfile +++ /dev/null @@ -1,89 +0,0 @@ -FROM almalinux:9.3-minimal-20231124 as base -LABEL Description="This is a Control-M/Agent image that planned to run in K8s env" - -RUN echo "Update OS packages to the latest version" \ - && microdnf update -y \ - && echo "The next packages used by Control-M/Agent" \ - && echo " binutils used for strings" \ - && echo " hostname used for hostname" \ - && echo " libnsl used for perl" \ - && echo " ncurses used for installer engine" \ - && echo " net-tools used for netstat and ss" \ - && echo " procps used for ps" \ - && echo " psmisc used for pstree" \ - && echo " which used for which" \ - && echo " sudo used for sudo" \ - && echo "Install required packages for Control-M/Agent" \ - && microdnf update -y \ - && microdnf install -y --nodocs --noplugins --setopt=install_weak_deps=0 \ - util-linux \ - psmisc \ - which \ - net-tools \ - hostname \ - binutils \ - jq \ - libnsl \ - ncurses \ - procps \ - shadow-utils \ - tar \ - tcsh \ - libxcrypt-compat \ - && microdnf install -y java-11-openjdk-headless --nodocs --noplugins --setopt=install_weak_deps=0 \ - && microdnf install -y python39 \ - && ln -s /usr/bin/python3.9 /usr/bin/python \ - && python3.9 -m ensurepip \ - && ln -s /usr/bin/pip3 /usr/bin/pip \ - && echo "Python version: " && python --version \ - && echo "Pip version: " && pip3 --version \ - && curl -sL -o /tmp/kubectl "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ - && install -o root -g root -m 0755 /tmp/kubectl /usr/local/bin/kubectl \ - && kubectl version --client || exit 1 \ - && java -version || exit 1 \ - && microdnf clean all - -# Add controlm user and modify sudoers file -ARG USERNAME=controlm -RUN echo "Create $USERNAME user" \ - && useradd -d /home/$USERNAME -s /bin/bash -g 0 -m $USERNAME \ - && chmod -R 775 /home/$USERNAME \ - && echo "add $USERNAME user and root to soduers list" \ - && echo "root ALL=(ALL) ALL" >> /etc/sudoers \ - && echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers - - -FROM base as intermediate -ARG USERNAME=controlm -USER root - -# install ctm-automation-api kitWORKDIR /root -WORKDIR /root - -RUN echo "Install Automation API CLI" \ - && mkdir ctm-automation-api \ - && cd ctm-automation-api \ - && curl -sO https://controlm-appdev.s3.us-west-2.amazonaws.com/deploy/9.22.10/install_ctm_cli.py \ - # && curl -sO https://s3-us-west-2.amazonaws.com/controlm-appdev/dev/shoshana/install_ctm_cli.py \ - && python install_ctm_cli.py \ - && ctm -v - -USER controlm -WORKDIR /home/controlm - -ARG AAPI_END_POINT -ARG AAPI_TOKEN -ARG AGENT_IMAGE_NAME=Agent.Linux -ENV BMC_INST_JAVA_HOME=/etc/alternatives/jre_11_openjdk -RUN echo "Provision Control-M/Agent" \ - && ctm env add tenant $AAPI_END_POINT $AAPI_TOKEN \ - && ctm provision image $AGENT_IMAGE_NAME || exit 1 \ - && ctm env del tenant \ - && echo "source .bash_profile" >> .bashrc \ - && cp -r BMCINSTALL/external_java_path* BMCINSTALL/scripts . && rm -rf BMCINSTALL/* && mv external_java_path* scripts BMCINSTALL/ - -ADD --chown=controlm:controlm resources/* /home/controlm/ -RUN chmod +x *.sh - -EXPOSE 7006 -ENTRYPOINT /bin/bash -c "./startup.sh $PERSISTENT_VOL $AAPI_END_POINT $AAPI_TOKEN $AGENT_TOKEN_TAG $AGENT_HOSTGROUP_NAME" \ No newline at end of file diff --git a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/docker/resources/ctmhost_keepalive.sh b/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/docker/resources/ctmhost_keepalive.sh deleted file mode 100644 index 05a49e5..0000000 --- a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/docker/resources/ctmhost_keepalive.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -source ~/.bash_profile - -echo "Checking Agent communication with Control-M Server" -ag_diag_comm -ctmaggetcm - -# ag_ping Loop control variables -# LIMIT is number of times to check HOSTS before timing out in error -LIMIT=5 -# SLEEPTIME is number of seconds to sleep between successive pings to CURRENT_HOST -SLEEPTIME=120 -COUNTER=0 - -# Create the HOSTS array -HOSTS=( $( grep CTMPERMHOSTS $CONTROLM/data/CONFIG.dat | awk '{ print $2 }' | tr '|' ' ' ) ) -IFS='|' -OIFS=$IFS -for h in $PERM_HOSTS; -do - HOSTS+=($h) -done -IFS=$OIFS - -# Begin the main ag_ping loop -until [[ "${COUNTER}" -eq "${LIMIT}" ]] -do - CURRENT_HOST=$(grep "CTMSHOST" $CONTROLM/data/CONFIG.dat | awk '{print $2}') - printf '%s %s\n' "$(date)" "Current Host is $CURRENT_HOST" - NEW_HOST="" - - ag_diag_comm - ag_ping > /dev/null 2>&1 - # If ag_ping fails check other HOSTS - if [[ $? -gt 0 ]]; - then - printf '%s %s\n' "$(date)" "Error: ag_ping to $CURRENT_HOST failed!" - for NEXT_HOST in ${HOSTS[@]}; - do - if [[ "$CURRENT_HOST" != "$NEXT_HOST" ]]; - then - printf '%s %s\n' "$(date)" "Switching to Host $NEXT_HOST" - sed -i -e "/CTMSHOST/s/$CURRENT_HOST/$NEXT_HOST/" $CONTROLM/data/CONFIG.dat - ag_diag_comm - sleep 5 - ag_ping > /dev/null 2>&1 - sleep 5 - ag_ping > /dev/null 2>&1 - if [[ $? -gt 0 ]] - then - printf '%s %s\n' "$(date)" "Error: ag_ping to $NEXT_HOST failed!" - continue - else - printf '%s %s\n' "$(date)" "New Host is $NEXT_HOST" - #reset the timeout counter - COUNTER=0 - NEW_HOST=$NEXT_HOST - break - fi - fi - done - if [[ -z "${NEW_HOST}" ]]; - then - #increase the timeout counter - COUNTER=$((COUNTER+1)) - #reset back to current_host - printf '%s %s\n' "$(date)" "All connections to PERM_HOSTS failed. Resetting to $CURRENT_HOST $COUNTER time" - sed -i -e "/CTMSHOST/s/$NEXT_HOST/$CURRENT_HOST/" $CONTROLM/data/CONFIG.dat - ag_diag_comm - sleep 5 - ag_ping > /dev/null 2>&1 - fi - else - COUNTER=0 - printf '%s %s\n' "$(date)" "Connected to $CURRENT_HOST" - sleep $SLEEPTIME - fi -done - -# We have reached our LIMIT number of tries. Agent cannot find a HOST. Alerting and abort -printf '%s %s\n' "$(date)" "Error: Timed out trying to connect to a CTM Host - aborting" -exit 99 diff --git a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/docker/resources/startup.sh b/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/docker/resources/startup.sh deleted file mode 100644 index dcbe6bf..0000000 --- a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/docker/resources/startup.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash -cd "$(dirname "$0")" -echo "Configures Agent container" - -echo "Install K8s certificates for AI server" -export CONTROLM=/home/controlm/ctm -PEM_FILE=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt -KEYSTORE=$CONTROLM/cm/AI/data/security/apcerts -KSPASS=appass - -NUM_CERTS=$(grep -c 'END CERTIFICATE' $PEM_FILE) - -for N in $(seq 0 $((NUM_CERTS - 1))); do - awk "n==$N { print }; /END CERTIFICATE/ { n++ }" $PEM_FILE | - keytool -noprompt -import -trustcacerts -alias "kube-pod-ca-$N" -keystore "$KEYSTORE" -storepass $KSPASS -done - -echo "Parameters: $argv" -AG_NODE_ID=$(hostname) -PERSISTENT_VOL=$1/$AG_NODE_ID -AAPI_END_POINT=$2 -AAPI_TOKEN=$3 -AGENT_TOKEN_TAG=$4 -CTM_SERVER_NAME=IN01 -AGENT_HOSTGROUP_NAME=$5 -FOLDERS_EXISTS=false -AGENT_REGISTERED=false - -# create if needed, and map agent persistent data folders -echo 'Mapping persistent volume' -source ~/.bash_profile - -if [ ! -d "$PERSISTENT_VOL"/pid ]; then - echo "Persistent connection : internal AR keep-alive" - { - echo "AR_PING_TO_SERVER_IND Y" - echo "AR_PING_TO_SERVER_INTERVAL 30" - echo "AR_PING_TO_SERVER_TIMEOUT 60" - echo "DISABLE_CM_SHUTDOWN Y" - } >>~/ctm/data/CONFIG.dat - touch ~/ctm/data/DISABLE_CM_SHUTDOWN_Y.cfg - - echo "Update Agent configuration file with current hostname" - ctmcfg -table CONFIG -action update -parameter INSTALL_HOSTNAME -value "${AG_NODE_ID}" - ctmcfg -table CONFIG -action update -parameter LOCALHOST -value "${AG_NODE_ID}" - ctmcfg -table CONFIG -action update -parameter PHYSICAL_UNIQUE_AGENT_NAME -value "${AG_NODE_ID}" - - # no agent files exist in PV, copy the current agent files to PV - echo 'first time the agent is using the persistent volume, moving folders to persistent volume' - mkdir "$PERSISTENT_VOL" - mv $CONTROLM/backup $CONTROLM/capdef $CONTROLM/dailylog $CONTROLM/data $CONTROLM/measure $CONTROLM/onstmt $CONTROLM/pid $CONTROLM/procid $CONTROLM/status $CONTROLM/sysout $CONTROLM/cm -t "$PERSISTENT_VOL" -else - echo 'this is not the first time an agent is running using this persistent volume, mapping folder to existing persistent volume' - FOLDERS_EXISTS=true - rm -Rf $CONTROLM/backup $CONTROLM/capdef $CONTROLM/dailylog $CONTROLM/data $CONTROLM/measure $CONTROLM/onstmt $CONTROLM/pid $CONTROLM/procid $CONTROLM/status $CONTROLM/sysout $CONTROLM/cm - sed '/CM_LIST_SENT2CTMS/d' "$PERSISTENT_VOL"/data/CONFIG.dat -fi -# create link to persistent volume -ln -s "$PERSISTENT_VOL"/backup $CONTROLM/backup -ln -s "$PERSISTENT_VOL"/capdef $CONTROLM/capdef -ln -s "$PERSISTENT_VOL"/dailylog $CONTROLM/dailylog -ln -s "$PERSISTENT_VOL"/data $CONTROLM/data -ln -s "$PERSISTENT_VOL"/measure $CONTROLM/measure -ln -s "$PERSISTENT_VOL"/onstmt $CONTROLM/onstmt -ln -s "$PERSISTENT_VOL"/pid $CONTROLM/pid -ln -s "$PERSISTENT_VOL"/procid $CONTROLM/procid -ln -s "$PERSISTENT_VOL"/sysout $CONTROLM/sysout -ln -s "$PERSISTENT_VOL"/status $CONTROLM/status -ln -s "$PERSISTENT_VOL"/cm $CONTROLM/cm - -echo "Using new AAPI configuration, not the default build time configuration" -ctm env add ctm_env "$AAPI_END_POINT" "$AAPI_TOKEN" -ctm env set ctm_env - -echo "Check if Agent exists in the Control-M Server" -if $FOLDERS_EXISTS ; then - ctm config server:agents::get "$CTM_SERVER_NAME" "$AG_NODE_ID" | grep "$AG_NODE_ID" - if [[ $? == "0" ]] ; then - echo 'agent already exists' - AGENT_REGISTERED=true - fi -fi - -if $FOLDERS_EXISTS && $AGENT_REGISTERED ; then - echo 'Starting the Agent' - start-ag -u controlm -p ALL -else - echo 'Configuring and registering the agent' - ctm provision saas:agent::setup $AGENT_TOKEN_TAG $AG_NODE_ID -fi - -echo 'Checking Agent communication with Control-M Server' -ag_ping -ag_diag_comm - -echo 'Deploying Control-M/Integration Kubernetes Job Type to Control-M/Agent' -ctm deploy ai:jobtype IN01 "$AG_NODE_ID" KUBERNETES | grep "successful" >res.txt - -x=1 -while [[ ! -s res.txt && $x -lt 20 ]]; do - echo "attempt #$x" - ctm deploy ai:jobtype IN01 "$AG_NODE_ID" KUBERNETES | grep "successful" >res.txt - x=$((x + 1)) - sleep 20 -done - -if [[ $x -lt 20 ]]; then - echo 'Deploying Control-M/Integration Kubernetes Job Type to Control-M/Agent successes' -else - echo 'Deploying Control-M/Integration Kubernetes Job Type to Control-M/Agent failed' -fi -rm res.txt - -echo 'KUBERNETES AI Job Type is ready for use' - -echo 'Adding the Agent to Host Group' -ctm config server:hostgroup:agent::add "$CTM_SERVER_NAME" "$AGENT_HOSTGROUP_NAME" "$AG_NODE_ID" - -bash ./ctmhost_keepalive.sh \ No newline at end of file diff --git a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/roles.yaml b/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/roles.yaml deleted file mode 100644 index 34991c2..0000000 --- a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/roles.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: node-apis ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: node-apis -rules: - - apiGroups: - - "" - - "apps" - - "batch" - resources: - - endpoints - - deployments - - pods - - pods/log - - jobs - verbs: - - get - - list - - watch - - create - - delete ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: node-apis - namespace: default -subjects: - - kind: ServiceAccount - name: node-apis - namespace: default -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: node-apis \ No newline at end of file diff --git a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/sleep_for_20sec_job.yaml b/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/sleep_for_20sec_job.yaml deleted file mode 100644 index 1b6f92f..0000000 --- a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/sleep_for_20sec_job.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: job.20.sec -spec: - template: - metadata: - name: job.20.sec - labels: { - app: controlm, - controlm-order-id-internal-label : controlm.00001.1 - } - - spec: - containers: - - name: ubuntu - image: ubuntu:latest - # Just spin & wait forever - command: [ "/bin/bash", "-c", "--" ] - args: [ "echo START-`date`; sleep 21; echo END-`date`" ] - restartPolicy: Never - - diff --git a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/stateful_ha.yaml b/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/stateful_ha.yaml deleted file mode 100644 index 5b11223..0000000 --- a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/stateful_ha.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: ctm-agent-pvc -spec: - accessModes: - - ReadWriteOnce - storageClassName: - resources: - requests: - storage: 10Gi ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - labels: - type: stateful-service-ctmag-srv - name: ctm-statefulset-agent -spec: - replicas: 1 - selector: - matchLabels: - type: stateful-service-ctmag-srv - serviceName: ctmag - template: - metadata: - labels: - type: stateful-service-ctmag-srv - spec: - containers: - - env: - - name: PERSISTENT_VOL - value: /home/controlm/persistent_folder - - name: AAPI_END_POINT - value: - - name: AAPI_TOKEN - valueFrom: - secretKeyRef: - name: credentials - key: AAPI_TOKEN - - name: AGENT_TOKEN_TAG - value: - - name: AGENT_HOSTGROUP_NAME - value: - image: - imagePullPolicy: Always - name: ctmagent-container - ports: - - containerPort: 7006 - volumeMounts: - - name: pv_data - mountPath: /home/controlm/persistent_folder - securityContext: - fsGroup: 1000 - terminationGracePeriodSeconds: 10 - volumes: - - name: pv-data - persistentVolumeClaim: - claimName: ctm-agent-pvc diff --git a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/testAIJob.json b/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/testAIJob.json deleted file mode 100644 index 9db8fbd..0000000 --- a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/testAIJob.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "Folder_Test_K8S": { - "Type": "Folder", - "ControlmServer": "IN01", - "AIKubernetes_Job_Test": { - "Type": "Job:ApplicationIntegrator:AI Kubernetes", - "ConnectionProfile": "TEST_KUBECTL", - "AI-Action": "Apply", - "AI-YamlFile": "/home/controlm/sleep_for_20sec_job.yaml", - "Host": "" - } - } -} \ No newline at end of file diff --git a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/testAiConnectionProfile.json b/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/testAiConnectionProfile.json deleted file mode 100644 index 8fd93f9..0000000 --- a/helix-control-m/301-statefulset-agent-to-run-k8s-jobs-using-ai-job/testAiConnectionProfile.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "TEST_KUBECTL": { - "Type": "ConnectionProfile:ApplicationIntegrator:AI Kubernetes", - "AI-Kubectl Location": "kubectl", - "AI-Run As:": "controlm", - "Description": "K8S connection profile example", - "Centralized": true, - "NoTestOnDeploy": true - } -} \ No newline at end of file diff --git a/helix-control-m/302-external-monitoring-tools-example/README.md b/helix-control-m/302-external-monitoring-tools-example/README.md deleted file mode 100644 index 8394a36..0000000 --- a/helix-control-m/302-external-monitoring-tools-example/README.md +++ /dev/null @@ -1,27 +0,0 @@ -## Customizable Alerts Listener Implementation - -Helix Control-M provides ability to integrate with external monitoring tools via web sockets. -The file alertsListener.ts is an example of a functional alerts listener that can be used to listen to alerts that come from the Control-M system. -The alertsListener uses websocket that is based on the ws library [ws GitHub](https://github.com/WebSocket/ws). -This is an alternative to the alert's listener client provided by BMC through ctm cli and described in [External Alerts Management](https://docs.bmc.com/docs/automation-api/Helix21/run-service-1041161864.html#Runservice-alerts_listener_startrunalerts:listener::start:~:text=Back%20to%20top-,External%20Alert%20Management,-An%20alert%20is). - -#### Running the script -```javascript -node alertsListener.js -``` - -The alertsListener client example reads the parameters from a Json file -### example for json file -```javascript -{ - "listenerScript": "C:\\Users\\user\\scriptExample.bat", - "listenerEnvironment": "listenerEnvironment", - "token": "userToken", - "url": "wss://user-63990-alerts.us1.ci.ctmsaas.com" -} -``` - -listenerScript - specify the path to the alerts script that runs against every alert, provided by the customer. -listenerEnvironment - specify the environment in which the alerts are configured -token - api token. -url - WebSocket Url for the connection. diff --git a/helix-control-m/302-external-monitoring-tools-example/WindowsService.md b/helix-control-m/302-external-monitoring-tools-example/WindowsService.md deleted file mode 100644 index 9bfc524..0000000 --- a/helix-control-m/302-external-monitoring-tools-example/WindowsService.md +++ /dev/null @@ -1,53 +0,0 @@ -## Configuring the Alerts Listener as a Windows Service - -This procedure describes how to configure an alerts_listener system service on a Windows operating system. -You can use this service instead of the Automation API commands to start or stop alerts listening. - -### Before You Begin - -Ensure successful completion of the following steps described in [Setting Up External Alerts](https://documents.bmc.com/supportu/dev-saas/en-US/Documentation/Alerts.htm#Setting). -1. Enable external alerts. -2. (Optional) Submit a template to configure alert details. -3. Set the Helix Control-M environment. -4. Submit a script with details of external alert handling - - -### Begin -Use a wrapper executable such as [WinSW](https://github.com/winsw/winsw) -(Windows Service Wrapper), Or [NSSM](https://nssm.cc/scenarios) -(Non-Sucking Service Manager) to register the ctm.cmd executable of -[Control-M Automation API CLI](https://docs.bmc.com/docs/saas-api/setting-up-the-api-946711372.html#SettinguptheAPI-ctm_cliInstallingtheControl-MAutomationAPICLI) -as a system service named alerts_listener, as follows: - -- #### WinSW - -Create a configuration file with the details of the service, as in the following example: -```xml - - alerts_listener - Control-M Alerts Listener - This service runs Control-M Alerts Listener - @{CTM_CLI_PATH}\ctm.cmd - run alerts:listener::start - @{CTM_CLI_PATH}\ctm.cmd - run alerts:listener::stop - Manual - - - %CTM_CLI_PROFILE_PATH%\logs - -``` -Submit your configuration file when you install the service by running the following command: -**alerts_listener install** - -- #### NSSM - -Install the **alerts_listener** service using the following command - -**nssm install alerts_listener** - -In the NSSM Service Installer, define the path to the **ctm.cmd** executable in the **Application** tab and the credentials of your Control-M user account in the **Log on** tab. -From a command line, run the **alerts_listener** service with one of the following commands: -- ***start*** -- ***stop*** - diff --git a/helix-control-m/302-external-monitoring-tools-example/alertsListener.ts b/helix-control-m/302-external-monitoring-tools-example/alertsListener.ts deleted file mode 100644 index 023bd70..0000000 --- a/helix-control-m/302-external-monitoring-tools-example/alertsListener.ts +++ /dev/null @@ -1,404 +0,0 @@ -// - -const webSocket = require('ws') -const HttpsProxyAgent = require('https-proxy-agent'); -const winston = require('winston'); -const path = require("path"); -const spawn = require('child_process').spawn -const fs = require('fs'); -const os = require("os"); - -const powerShellExtension = ".ps1"; -let headers; -let logger; -let environmentArgument; -let scriptPathArgument; -let apiKeyArgument; -let UrlArgument; -let proxyArgument; - -export function getLoggerFileName() { - return path.join(ctmDir(), "logs", `alertsListener_${getCurrentTime()}.log`); -} - -export function createAlertsDir(dirPath: string) { - let logFolder = path.join(dirPath, "logs"); - if (!fs.existsSync(logFolder)) - fs.mkdirSync(logFolder) -} - -export function switchToScriptDir(scriptPathArgument: string) { - try { - let dirPath = path.dirname(scriptPathArgument); - logger.info("Switching to: " + dirPath); - process.chdir(dirPath); - } catch (error) { - logger.error("Could not switch to: " + scriptPathArgument + " directory, reason: " + error); - throw new Error(error); - } -} - -// This function creates the webSocket instance and handling the following events : open, close, error and message. -// There is a check via ping that ensures the server is up. -//each message that is received from the server is being logged and modified in order to run as parameters for the given script. -export function connect(url, headers, delay) { - let ws = new webSocket(url, [], headers); - ws.isAlive = true; - ws.shouldClose = false; - let pingInterval; - ws.onopen = function () { - handleLogsDir() - pingInterval = createPingInterval(url, ws, delay) - }; - - ws.onmessage = function (e) { - logger.info(e.data); - const scriptArgumentsArray = [scriptPathArgument]; - createScriptPath(JSON.parse(e.data), scriptArgumentsArray); - if (checkScriptExists(scriptPathArgument)) { - switchToScriptDir(scriptPathArgument); - let shell = addingShellRelatedCommand(scriptArgumentsArray); - logger.debug(`running the following command: ${scriptArgumentsArray.join(' ')}`) - runScript(shell, scriptArgumentsArray); - } else { - ws.terminate() - } - }; - - ws.onclose = function (e) { - logger.info("Connection closed for the following reason: ", e.message) - if (ws.shouldClose === true) { - logger.info("closing the Alerts listener client."); - clearInterval(pingInterval); - } else { - logger.info("reconnecting to the API gateway."); - clearInterval(pingInterval); - setTimeout(function () { - connect(url, headers, delay) - }, 1000 * 20) - } - }; - - ws.onerror = function (err) { - logger.error(`${err.message} Closing socket`); - logger.debug(JSON.stringify(err.message)) - let errorCode = extractErrorCode(err.message); - handleErrorCode(errorCode, ws); - }; - - ws.on('pong', heartbeat) - return ws; -} - -function runScript(shell: string, scriptArgumentsArray: any[]) { - let childProcess = spawn(shell, scriptArgumentsArray); - childProcess.stdout.on('data', (data) => { - logger.debug(`script output: ${data}`); - }); - childProcess.stderr.on('data', (data) => { - logger.error(`script error message: ${data}`); - }); -} - -export function checkScriptExists(scriptPath: string): boolean { - try { - validateListenerScriptPath(scriptPath) - } catch (error) { - logger.error(error); - return false; - } - return true; -} - -// Restricts the amount of files in the log's directory, keeps the latest update logs, default 10 files. -function handleLogsDir() { - const alertLogDir = path.join(ctmDir(), "logs"); - const files = fs.readdirSync(alertLogDir); - const maxFilesCount = parseInt(getSettings("listenerMaxFileCount", "10")) - if (files.length <= maxFilesCount) { - return; - } - const fileToDateObject = []; - files.forEach(file => { - getFileLastModifiedDate(file, alertLogDir, fileToDateObject); - }) - sortAndRemoveOldFiles(fileToDateObject, maxFilesCount); -} - -export function getFileLastModifiedDate(file: string, alertLogDir, fileToDateObject: any[]) { - let filePath = path.join(alertLogDir, file); - let fileCreation = fs.statSync(filePath).mtime.getTime(); - fileToDateObject.push({fileCreation, filePath}) -} - -export function sortAndRemoveOldFiles(fileToDateObject: any[], maxFilesCount: number) { - fileToDateObject.sort(function (a, b) { - return a.fileCreation - b.fileCreation - }) - while (fileToDateObject.length > maxFilesCount) { - fs.unlinkSync(fileToDateObject[0].filePath); - fileToDateObject.shift() - } -} - -function heartbeat() { - logger.debug("pong received") - this.isAlive = true; -} - -function addArgumentsToScript(key: string[], scriptArgumentsArray: any[], keyValue: string) { - key[0] = key[0] + ": "; - scriptArgumentsArray.push(key[0], keyValue) -} - -export function createScriptPath(object, scriptArgumentsArray ?: any[]) { - for (let i = 0; i < object.alertFields.length; ++i) { - let field = object.alertFields[i]; - let key = Object.keys(field) - if (powerShellExtension === path.extname(scriptArgumentsArray[0])) { - let keyValue = `'${field[key.toString()].replace((/'/g, "\\'"))}'` + " "; - addArgumentsToScript(key, scriptArgumentsArray, keyValue); - } else { - let keyValue = field[key.toString()] + " "; - addArgumentsToScript(key, scriptArgumentsArray, keyValue); - } - } -} - -function addingShellRelatedCommand(commandArgs: any[]) { - let shell = isWindows() ? 'cmd' : '/bin/sh'; - logger.debug("shell is: " + shell); - if (isWindows()) { - if (powerShellExtension !== path.extname(commandArgs[0])) { - commandArgs.splice(0, 0, '/C'); - } else { - shell = "powershell"; - } - } - return shell; -} - -function isWindows() { - return os.platform() === "win32"; -} - -function handleError(ws, message: string) { - logger.error(message); - ws.shouldClose = true; -} - -function createPingInterval(url: string, ws, delay: number) { - return setInterval(function () { - if (ws.isAlive === false) { - logger.debug("Did not receive pong from server. Trying to reconnect") - ws.terminate() - connect(url, headers, delay); - } - ws.isAlive = false; - logger.debug("WebSocket sending ping") - ws.ping() - }, delay); -} - -function handleErrorCode(errorCode: number, ws) { - switch (errorCode) { - case 401 : - handleError(ws, "User is not authorized to perform this operation"); - break - case 403 : - handleError(ws, "User is not permitted to perform this operation"); - break - case 409 : - handleError(ws, "Another Alerts listener is already connected"); - break - case 503 : - handleError(ws, "External Alerts Service is disabled"); - break - default : - ws.close(); - logger.info('Socket is closed. Trying to reconnect.'); - } -} - -function extractErrorCode(errorMessage) { - return parseInt(errorMessage.split(':')[1]); -} - -function getCurrentTime(logFormat ?: boolean) { - const separator = logFormat ? ":" : "-"; - const today = new Date(); - return today.getFullYear() + '-' + (padTime(today.getMonth() + 1)) + '-' + padTime(today.getDate()) - + '-' + padTime(today.getHours()) + separator + padTime(today.getMinutes()) + separator + padTime(today.getSeconds()); -} - -function padTime(time: number) { - return (time.toString() as any).padStart(2, '0') -} - -export function validateScriptAndEnvArgs(scriptPathArgument: string, environmentArgument: string) { - if (scriptPathArgument.length === 0) { - throw new Error("Missing script path") - } - if (environmentArgument.length === 0) { - throw new Error("listenerEnvironment not found") - } -} - -export function validateTokenAndUrlArgs(apiKeyArgument, UrlArgument) { - if (apiKeyArgument.length === 0) { - throw new Error("Missing Api Key") - } - if (UrlArgument.length === 0) { - throw new Error("Missing URL") - } -} - -function parseArgsAndValidate() { - scriptPathArgument = getSettings("listenerScript", "") - environmentArgument = getSettings("listenerEnvironment", "") - validateScriptAndEnvArgs(scriptPathArgument, environmentArgument) - apiKeyArgument = getSettings("token", "") - UrlArgument = getSettings("url", ""); - validateTokenAndUrlArgs(apiKeyArgument, UrlArgument); - proxyArgument = getSettings("proxyUrl", ""); - -} - -function printArgs() { - logger.info("process pid is: " + getSettings("listenerPid", "")); - logger.info("Running WebSocket with the following URL: " + UrlArgument, "and the following script path: " + scriptPathArgument + " against the following environment: " + environmentArgument) -} - -// If proxyUrl is defined, websocket will try to connect to the given proxyUrl -function checkProxySettings(headers, proxyUrl: string) { - if (proxyUrl !== undefined && proxyUrl.length !== 0) { - headers.agent = new HttpsProxyAgent(proxyUrl); - logger.debug("running through proxy environment:" + proxyUrl); - } -} - -// Creates the log's directory if needed, preparing the arguments for the webSocket. -export function init() { - createAlertsDir(ctmDir()) - setLogger(createLogger()); - parseArgsAndValidate() - printArgs() - headers = {headers: {['x-api-key']: apiKeyArgument}}; - checkProxySettings(headers, proxyArgument); - try { - connect(UrlArgument, headers, 1000 * 60); - } catch (e) { - logger.info("caught exception ") - } -} - -// Creating winston logger, for more info about the flags and configuration see - https://www.npmjs.com/package/winston/v/1.1.2 -export function createLogger() { - return new (winston.Logger)({ - transports: [ - new (winston.transports.File)({ - json: false, - filename: getLoggerFileName(), - timestamp: function () { - return getCurrentTime(true); - }, - formatter: (options) => { - return options.timestamp() + ' ' + options.level.toUpperCase() + ' ' + (options.message ? options.message : ''); - }, - level: getLogLevel(), - maxsize: getFileMaxSize() - }), - ] - }); -} - -export function setLogger(loggerInstance: any) { - logger = loggerInstance; -} - -function getLogLevel() { - return getSettings("listenerLogLevel", "info"); -} - -function getFileMaxSize() { - return parseInt(getSettings("listenerLogSizeMb", "10")) * Math.pow(10, 6); -} - -function ctmDir() { - const dir = process.env.CTM_CLI_PROFILE_PATH || - path.join(homeFolder(), ".ctm"); - - if (!pathExistsSync(dir)) { - fs.mkdirSync(dir, 502); // 0766 - } - - return dir; -} - -function getSettings(settingName: string, defaultValue: string): string { - const config = getCliConfigurations(); - const result = config[settingName]; - return result === undefined ? defaultValue : result; -} - -function getCliConfigurations() { - const confFile = getConfigurationFile(); - const ret = readJsonFile(confFile); - return ret || {localServerLocation: ""}; -} - -function getConfigurationFile() { - return path.join(ctmDir(), "settingsFile.json"); -} - -function readJsonFile(file: string): any { - try { - return require(file); - } catch (error) { - return undefined; - } -} - -function homeFolder() { - if (process.env.HOME !== undefined) { - return process.env.HOME; - } - - if (os.homedir()) { - return os.homedir() - } - - if (process.env.HOMEDRIVE && process.env.HOMEPATH) { - return process.env.HOMEDRIVE + process.env.HOMEPATH; - } - - throw new Error("No HOME path available"); -} - -function pathExistsSync(path: string) { - try { // see http://stackoverflow.com/questions/4482686 - fs.statSync(path); - return true; - } catch (err) { - return false; - } -} - -function validateListenerScriptPath(path: string) { - if (fs.existsSync(path)) { - try { - fs.readFileSync(path) - } catch (error) { - throw new Error("Alerts listener script path: " + path + " must be a file"); - } - try { - fs.accessSync(path, fs.X_OK) - } catch (error) { - throw new Error("You are not authorized to execute Alerts listener script " + path); - } - } else { - throw new Error("Alerts listener script: " + path + " file not found"); - } -} - -init();