few extra changes, incorrect arg, use append for lists

Author: alemsh

condor_history_to_es.py

@@ -88,15 +88,15 @@ def es_import(document_generator):
     return success
 
 failed = False
-if options.access_points:
+if options.access_points and options.collectors:
     for coll_address in options.positionals:
         try:
             gen = es_generator(read_from_collector(coll_address, options.access_points, history=True))
             success = es_import(gen)
         except htcondor.HTCondorIOError as e:
             failed = e
             logging.error('Condor error', exc_info=True)
-if options.collectors:
+elif options.collectors:
     for coll_address in options.positionals:
         try:
             gen = es_generator(read_from_collector(coll_address, history=True))

condor_history_to_prometheus.py

@@ -38,12 +38,12 @@ def locate_schedds(collector, access_points):
     if access_points:
         try:
             for ap in access_points:
-                schedds += coll.locate(htcondor.DaemonTypes.Schedd, ap)
+                schedds.append(coll.locate(htcondor.DaemonTypes.Schedd, ap))
         except htcondor.HTCondorIOError as e:
             logging.error(f'Condor error: {e}')
     else:
         try:
-            schedds += coll.locateAll(htcondor.DaemonTypes.Schedd)
+            schedds.append(coll.locateAll(htcondor.DaemonTypes.Schedd))
         except htcondor.HTCondorIOError as e:
             logging.error(f'Condor error: {e}')
 
@@ -179,7 +179,7 @@ def iterate_ads(ads, name, metrics, last_job):
     # TODO: Add file tail function for condor history files
     #parser.add_option('-f','--histfile',
     #                help='history file to read from')
-    parser.add_option('-s','--schedd',default=False, action='store_true')
+    parser.add_option('-a','--access_points',default=None)
     parser.add_option('-p','--port', default=9100,
                     action='store', type='int',
                     help='port number for prometheus exporter')
@@ -208,7 +208,7 @@ def iterate_ads(ads, name, metrics, last_job):
         while True:
             start = time.time()
             for collector in args:
-                query_collector(collector, metrics, last_job)
+                query_collector(collector, options.access_points,  metrics, last_job)
 
             delta = time.time() - start
             # sleep for interval minus scrape duration

condor_queue_to_es.py

@@ -84,7 +84,7 @@ def es_generator(entries):
 es_import = partial(bulk, es, max_retries=20, initial_backoff=10, max_backoff=3600)
 
 failed = False
-if options.access_points:
+if options.access_points and options.collectors:
     for coll_address in options.positionals:
         try:
             gen = es_generator(read_from_collector(coll_address, options.access_points))

condor_queue_to_prometheus.py

@@ -75,7 +75,7 @@ def compose_ad_metrics(ads):
 
     parser.add_option('-c','--collectors',default=False, action='store_true',
                     help='read history from')
-
+    parser.add_option('-a','--access_points',default=None)
     parser.add_option('-p','--port', default=9100,
                     action='store', type='int',
                     help='port number for prometheus exporter')
@@ -97,14 +97,14 @@ def compose_ad_metrics(ads):
     while True:
         gens = []
         start = time.time()
-        if options.access_points:
+        if options.access_points and options.collectors:
             for coll_address in args:
                 try:
                     gens.append(read_from_collector(coll_address, options.access_points))
                 except htcondor.HTCondorIOError as e:
                     failed = e
                     logging.error('Condor error', exc_info=True)
-        if options.collectors:
+        elif options.collectors:
             for coll_address in args:
                 try:
                     gens.append(read_from_collector(coll_address))

condor_status_to_es.py

@@ -35,7 +35,7 @@
 class Dry:
     """Helper class for debugging"""
     _dryrun = False
-    
+
     def __init__(self, func):
         self.func = func
 
@@ -44,7 +44,7 @@ def __call__(self, *args, **kwargs):
             logging.info(self.func.__name__)
             logging.info(args)
             logging.info(kwargs)
-            
+
         else:
             return self.func(*args,**kwargs)
 
@@ -296,7 +296,7 @@ def key = status+"."+resource;
         "--after", default=timedelta(hours=1), help="time to look back", type=parse_time
     )
     parser.add_argument(
-        "-y",
+        "-d",
         "--dry-run",
         default=False,
         action="store_true",
@@ -353,7 +353,7 @@ def key = status+"."+resource;
 
     url = "{}://{}".format(prefix, address)
     logging.info("connecting to ES at %s", url)
-    es = Elasticsearch(hosts=[url], 
+    es = Elasticsearch(hosts=[url],
                     timeout=5000,
                     bearer_auth=token,
                     sniff_on_node_failure=True)

condor_utils.py

@@ -774,8 +774,8 @@ def read_from_collector(address, access_points=None, history=False, constraint='
     coll = htcondor.Collector(address)
     schedd_ads = []
     if access_points:
-        for ap in access_points:
-            schedd_ads += coll.locate(htcondor.DaemonTypes.Schedd, ap)
+        for ap in access_points.split(','):
+            schedd_ads.append(coll.locate(htcondor.DaemonTypes.Schedd, ap))
     else:
         schedd_ads = coll.locateAll(htcondor.DaemonTypes.Schedd)
     for schedd_ad in schedd_ads: