Reference for the technical implementation of the src.Cache project code.

Cache

Bases: NebulaBase

This module relies on the Redis in memory data store with expiry of cache enabled.

Source code in src/Cache.py
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
class Cache(NebulaBase):

    """
    This module relies on the Redis in memory data store with expiry of cache enabled.

    """

    def __init__(
        self,
    ):
        """
        Inorder to make Cache rest friendly replaced sys.exit() with raising exceptions which will get excepted
        in gustavo.py and eventually return a dictionary there {"error": True, "response": reason for error}
        """

        click.echo(
            click.style(
                "WARNING:This is an experimental feature and is not optimized for scale. Results might vary.",
                fg="red",
            )
        )
        NebulaBase.__init__(self)
        try:
            click.echo(
                click.style("CACHE_PREFIX:" + str(self.CACHE_PREFIX), fg="yellow")
            )
            self.redisObj = redis.StrictRedis(
                host=self.REDIS_IP, port=self.REDIS_PORT, password=self.REDIS_AUTH_TOKEN
            )
        except Exception as e:
            click.echo(click.style(e, fg="red"))
            # sys.exit()
            raise ErrorHandling

    def keyPartition(self, raw_key):
        """
        Partitions key based on the delimiter "_"

        Parameters
        ----------
        raw_key : string
            Raw key to be partitioned

        Returns
        -------

        component_tuple : list
            A tuple of three key components

        """
        key = str(raw_key).replace("'", "")
        key_components = str(key).split("_")
        return key_components[0], key_components[1], key_components[2]

    def scanLatest(
        self,
    ):

        """
        Iteratively scan and find the latest matching key and return the most freshest key

        Returns
        -------
        host_dict : dict
            A dictionary containing the latest key value pairs
        """

        host_dict = {}

        for key in self.redisObj.scan_iter(self.CACHE_PREFIX + "_*"):
            prefix, timestamp, host = self.keyPartition(key)
            if host not in host_dict.keys():
                host_dict[host] = timestamp
            else:
                if host_dict[host] < timestamp:
                    host_dict[host] = timestamp
        return host_dict

    def getHostDeviceGroupFromKey(self, host_id):
        """
        Obtain the device group from the given key

        Parameters
        ----------

        host_id : string
            The key value to check

        Returns
        -------

        host : string
            host part of the key

        device_group : string
            device group part of the key

        """
        device_group = host_id.split("@")[0]
        host = host_id.split("@")[1]
        return host, device_group

    def getHosts(self, device_group_queried, host_queried):

        """
        Determines which hosts and device groups currently exist in the cache by querying Redis.

        Parameters
        ----------
        device_group_queried : string
            The device group being queried

        host_queried : string
            The host being queried

        Returns
        -------
        mapping_dict : dict
            A dict containing {
                                "host_queried":the query results for the host,
                                "device_group_queried":the query results for the device group,
                                "response":a boolean value depending on success of query
                              }


        """

        host_dict = {}
        device_group_dict = {}
        for key in self.redisObj.scan_iter(self.CACHE_PREFIX + "_*"):
            prefix, timestamp, host_id = self.keyPartition(key)
            host, device_group = self.getHostDeviceGroupFromKey(host_id=host_id)
            if host not in host_dict.keys():
                host_dict[host] = [device_group]
            else:
                host_dict[host] = host_dict[host] + [device_group]

            if device_group not in device_group_dict.keys():
                device_group_dict[device_group] = [device_group]
            else:
                device_group_dict[device_group] = device_group_dict[device_group] + [
                    device_group
                ]

        if host_queried == "all" and device_group_queried == "all":
            response = host_dict

        elif host_queried == "all" and device_group_queried != "all":
            if device_group_queried in device_group_dict.keys():
                response = device_group_dict[device_group_queried]
            else:
                response = []

        elif host_queried != "all" and device_group_queried == "all":
            if host_queried in host_dict:
                # print(host_dict[host_queried])
                response = host_dict[host_queried]
            else:
                response = []

        else:
            if (
                host_queried in host_dict.keys()
                and device_group_queried in host_dict[host_queried]
            ):
                # print(True)
                response = True
            else:
                # print(False)
                response = False

        return {
            "host_queried": host_queried,
            "device_group_queried": device_group_queried,
            "response": response,
        }

    def unpickleData(self, device_group, host):

        """
        Given a device group and host, unpickles the query result (vitals and containers) from Redis cache.

        Parameters
        ----------
        device_group: string
            The device group being queried

        host : string
            The host being queried

        Returns
        -------
        mapping_dict : dict
            A dict containing {
                                "host_queried":the query results for the host,
                                "device_group_queried":the query results for the device group,
                                "response":a boolean value depending on success of query
                              }

        key : string
            The key that led to the match

        """

        key = str(device_group + "@" + host)
        host_dict = self.scanLatest()
        # response = {"host_queried": host, "device_group_queried": device_group, "response": response}
        if key in host_dict.keys():
            timestamp = host_dict[key]
            try:
                dataObj = self.redisObj.get(
                    self.CACHE_PREFIX + "_" + timestamp + "_" + key
                )
                data_dict = pickle.loads(dataObj)
                return {
                    "host_queried": host,
                    "device_group_queried": device_group,
                    "response": data_dict,
                }, key
            except Exception as e:
                click.echo(click.style(e, fg="red"))
                return {
                    "host_queried": host,
                    "device_group_queried": device_group,
                    "response": {},
                }, key
        else:
            click.echo(
                click.style("{} not found in cache reports".format(key), fg="red")
            )
            return {
                "host_queried": host,
                "device_group_queried": device_group,
                "response": {},
            }, key

    def getIndividualVitals(self, device_group, host):
        """
        Fetches the vitals across device groups and host combinations
        Parameters
        ----------
        device_group: string
            The device group being queried

        host : string
            The host being queried

        TODO: REST-fy this function, currently executes sys.exit()

        Inorder to make Cache rest friendly replaced sys.exit() with raising exceptions which will get excepted
        in getAssetsForAll(self,asset,device_group_id="all",host_id="all") and eventually return a dictionary there
        {"error": True, "response": reason for error}

        """
        response, key = self.unpickleData(device_group, host)
        data_dict = response["response"]
        if bool(data_dict):
            try:
                mem = str(data_dict["memory_usage"])
                disk = str(data_dict["root_disk_usage"])
                cpu_core_use = str(data_dict["cpu_usage"]["cores"])
                cpu_pct_use = str(data_dict["cpu_usage"]["used_percent"])
                time = str(data_dict["report_creation_time"])
                click.echo(
                    click.style(
                        key
                        + "at time:"
                        + time
                        + "\t mem:"
                        + mem
                        + "\t"
                        + "disk:"
                        + disk
                        + "\t"
                        + "cpu_cores:"
                        + cpu_core_use
                        + "\t"
                        + "cpu_percent:"
                        + cpu_pct_use,
                        fg="blue",
                    )
                )
                return {
                    "error": False,
                    "response": key
                    + "at time:"
                    + time
                    + "\t mem:"
                    + mem
                    + "\t"
                    + "disk:"
                    + disk
                    + "\t"
                    + "cpu_cores:"
                    + cpu_core_use
                    + "\t"
                    + "cpu_percent:"
                    + cpu_pct_use,
                }

            except Exception as e:
                click.echo(click.style(e, fg="red"))
                # sys.exit()
                raise ErrorHandling
        else:
            click.echo(click.style("No key matches {}".format(key), fg="red"))
            return {"error": True, "response": "no key matches {}".format(key)}

    def getIndividualContainers(self, device_group, host):
        """
        Fetches the containers for device group and host combination
        Parameters
        ----------
        device_group: string
            The device group being queried

        host : string
            The host being queried

        TODO: REST-fy this function, currently executes sys.exit()

        Inorder to make Cache rest friendly replaced sys.exit() with raising exceptions which will get excepted
        in getAssetsForAll(self,asset,device_group_id="all",host_id="all") and eventually return a dictionary there
        {"error": True, "response": reason for error}
        """

        response, key = self.unpickleData(device_group, host)
        data_dict = response["response"]
        if bool(data_dict):
            try:
                containers = str(data_dict["apps_containers"])
                time = str(data_dict["report_creation_time"])
                click.echo(
                    click.style(
                        key + " at time:" + time + " containers:" + str(containers),
                        fg="blue",
                    )
                )
                return {
                    "error": False,
                    "response": key
                    + " at time:"
                    + time
                    + " containers:"
                    + str(containers),
                }
            except Exception as e:
                click.echo(click.style(e, fg="red"))
                # sys.exit()
                raise ErrorHandling
        else:
            click.echo(click.style("No key matches {}".format(key), fg="red"))
            return {"error": True, "response": "no key matches {}".format(key)}

    # not optimized at all

    def getAssetsForAll(self, asset, device_group_id="all", host_id="all"):
        """
        TODO: This function hasnt been implemented completely yet. Need to find an efficient way for querying at scale.
        """
        if device_group_id != "all" and host_id != "all":
            if asset == "vitals":
                try:
                    responseVitals = self.getIndividualVitals(device_group_id, host_id)
                except ErrorHandling:
                    return {
                        "error": True,
                        "response": "some problem with gathering vitals",
                    }
                except Exception as e:
                    return {"error": True, "response": e}
                return {"error": False, "response": responseVitals}
            elif asset == "containers":
                try:
                    responseContainers = self.getIndividualContainers(
                        device_group_id, host_id
                    )
                except ErrorHandling:
                    return {"error": True, "response": "some problem with containers"}
                except Exception as e:
                    return {"error": True, "response": e}
                return {"error": False, "response": responseContainers}

        host_dict = self.scanLatest()

        if len(host_dict.keys()) == 0:
            click.echo(
                click.style(
                    "No data matches the query device_group:{},hosts:{}".format(
                        device_group_id, host_id
                    ),
                    fg="red",
                )
            )
            return {
                "error": False,
                "response": "No data matches the query device_group:{},hosts:{}".format(
                    device_group_id, host_id
                ),
            }

        for host in host_dict.keys():
            device_group = host.split("@")[0]
            host = host.split("@")[1]
            fetch = False
            if device_group_id != "all" and device_group == device_group_id:
                fetch = True
            elif host_id != "all" and host == host_id:
                fetch = True
            elif host_id == "all" and device_group_id == "all":
                fetch = True

            if fetch:
                if asset == "vitals":
                    try:
                        responseVitals = self.getIndividualVitals(device_group, host)
                    except ErrorHandling:
                        return {
                            "error": True,
                            "response": "some problem with gathering vitals",
                        }
                    except Exception as e:
                        return {"error": True, "response": e}
                    return {"error": False, "response": responseVitals}
                elif asset == "containers":
                    try:
                        responseContainers = self.getIndividualContainers(
                            device_group, host
                        )
                    except ErrorHandling:
                        return {
                            "error": True,
                            "response": "some problem with containers",
                        }
                    except Exception as e:
                        return {"error": True, "response": e}
                    return {"error": False, "response": responseContainers}

        # return

__init__()

Inorder to make Cache rest friendly replaced sys.exit() with raising exceptions which will get excepted in gustavo.py and eventually return a dictionary there {"error": True, "response": reason for error}

Source code in src/Cache.py
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
def __init__(
    self,
):
    """
    Inorder to make Cache rest friendly replaced sys.exit() with raising exceptions which will get excepted
    in gustavo.py and eventually return a dictionary there {"error": True, "response": reason for error}
    """

    click.echo(
        click.style(
            "WARNING:This is an experimental feature and is not optimized for scale. Results might vary.",
            fg="red",
        )
    )
    NebulaBase.__init__(self)
    try:
        click.echo(
            click.style("CACHE_PREFIX:" + str(self.CACHE_PREFIX), fg="yellow")
        )
        self.redisObj = redis.StrictRedis(
            host=self.REDIS_IP, port=self.REDIS_PORT, password=self.REDIS_AUTH_TOKEN
        )
    except Exception as e:
        click.echo(click.style(e, fg="red"))
        # sys.exit()
        raise ErrorHandling

getAssetsForAll(asset, device_group_id='all', host_id='all')

TODO: This function hasnt been implemented completely yet. Need to find an efficient way for querying at scale.

Source code in src/Cache.py
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
def getAssetsForAll(self, asset, device_group_id="all", host_id="all"):
    """
    TODO: This function hasnt been implemented completely yet. Need to find an efficient way for querying at scale.
    """
    if device_group_id != "all" and host_id != "all":
        if asset == "vitals":
            try:
                responseVitals = self.getIndividualVitals(device_group_id, host_id)
            except ErrorHandling:
                return {
                    "error": True,
                    "response": "some problem with gathering vitals",
                }
            except Exception as e:
                return {"error": True, "response": e}
            return {"error": False, "response": responseVitals}
        elif asset == "containers":
            try:
                responseContainers = self.getIndividualContainers(
                    device_group_id, host_id
                )
            except ErrorHandling:
                return {"error": True, "response": "some problem with containers"}
            except Exception as e:
                return {"error": True, "response": e}
            return {"error": False, "response": responseContainers}

    host_dict = self.scanLatest()

    if len(host_dict.keys()) == 0:
        click.echo(
            click.style(
                "No data matches the query device_group:{},hosts:{}".format(
                    device_group_id, host_id
                ),
                fg="red",
            )
        )
        return {
            "error": False,
            "response": "No data matches the query device_group:{},hosts:{}".format(
                device_group_id, host_id
            ),
        }

    for host in host_dict.keys():
        device_group = host.split("@")[0]
        host = host.split("@")[1]
        fetch = False
        if device_group_id != "all" and device_group == device_group_id:
            fetch = True
        elif host_id != "all" and host == host_id:
            fetch = True
        elif host_id == "all" and device_group_id == "all":
            fetch = True

        if fetch:
            if asset == "vitals":
                try:
                    responseVitals = self.getIndividualVitals(device_group, host)
                except ErrorHandling:
                    return {
                        "error": True,
                        "response": "some problem with gathering vitals",
                    }
                except Exception as e:
                    return {"error": True, "response": e}
                return {"error": False, "response": responseVitals}
            elif asset == "containers":
                try:
                    responseContainers = self.getIndividualContainers(
                        device_group, host
                    )
                except ErrorHandling:
                    return {
                        "error": True,
                        "response": "some problem with containers",
                    }
                except Exception as e:
                    return {"error": True, "response": e}
                return {"error": False, "response": responseContainers}

    # return

getHostDeviceGroupFromKey(host_id)

Obtain the device group from the given key

Parameters:

Name Type Description Default
host_id string

The key value to check

required

Returns:

Name Type Description
host string

host part of the key

device_group : string device group part of the key

Source code in src/Cache.py
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
def getHostDeviceGroupFromKey(self, host_id):
    """
    Obtain the device group from the given key

    Parameters
    ----------

    host_id : string
        The key value to check

    Returns
    -------

    host : string
        host part of the key

    device_group : string
        device group part of the key

    """
    device_group = host_id.split("@")[0]
    host = host_id.split("@")[1]
    return host, device_group

getHosts(device_group_queried, host_queried)

Determines which hosts and device groups currently exist in the cache by querying Redis.

Parameters:

Name Type Description Default
device_group_queried string

The device group being queried

required

host_queried : string The host being queried

Returns:

Name Type Description
mapping_dict dict

A dict containing { "host_queried":the query results for the host, "device_group_queried":the query results for the device group, "response":a boolean value depending on success of query }

Source code in src/Cache.py
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
def getHosts(self, device_group_queried, host_queried):

    """
    Determines which hosts and device groups currently exist in the cache by querying Redis.

    Parameters
    ----------
    device_group_queried : string
        The device group being queried

    host_queried : string
        The host being queried

    Returns
    -------
    mapping_dict : dict
        A dict containing {
                            "host_queried":the query results for the host,
                            "device_group_queried":the query results for the device group,
                            "response":a boolean value depending on success of query
                          }


    """

    host_dict = {}
    device_group_dict = {}
    for key in self.redisObj.scan_iter(self.CACHE_PREFIX + "_*"):
        prefix, timestamp, host_id = self.keyPartition(key)
        host, device_group = self.getHostDeviceGroupFromKey(host_id=host_id)
        if host not in host_dict.keys():
            host_dict[host] = [device_group]
        else:
            host_dict[host] = host_dict[host] + [device_group]

        if device_group not in device_group_dict.keys():
            device_group_dict[device_group] = [device_group]
        else:
            device_group_dict[device_group] = device_group_dict[device_group] + [
                device_group
            ]

    if host_queried == "all" and device_group_queried == "all":
        response = host_dict

    elif host_queried == "all" and device_group_queried != "all":
        if device_group_queried in device_group_dict.keys():
            response = device_group_dict[device_group_queried]
        else:
            response = []

    elif host_queried != "all" and device_group_queried == "all":
        if host_queried in host_dict:
            # print(host_dict[host_queried])
            response = host_dict[host_queried]
        else:
            response = []

    else:
        if (
            host_queried in host_dict.keys()
            and device_group_queried in host_dict[host_queried]
        ):
            # print(True)
            response = True
        else:
            # print(False)
            response = False

    return {
        "host_queried": host_queried,
        "device_group_queried": device_group_queried,
        "response": response,
    }

getIndividualContainers(device_group, host)

Fetches the containers for device group and host combination

Parameters:

Name Type Description Default
device_group

The device group being queried

required

host : string The host being queried

TODO: REST-fy this function, currently executes sys.exit()

Inorder to make Cache rest friendly replaced sys.exit() with raising exceptions which will get excepted in getAssetsForAll(self,asset,device_group_id="all",host_id="all") and eventually return a dictionary there {"error": True, "response": reason for error}

Source code in src/Cache.py
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
def getIndividualContainers(self, device_group, host):
    """
    Fetches the containers for device group and host combination
    Parameters
    ----------
    device_group: string
        The device group being queried

    host : string
        The host being queried

    TODO: REST-fy this function, currently executes sys.exit()

    Inorder to make Cache rest friendly replaced sys.exit() with raising exceptions which will get excepted
    in getAssetsForAll(self,asset,device_group_id="all",host_id="all") and eventually return a dictionary there
    {"error": True, "response": reason for error}
    """

    response, key = self.unpickleData(device_group, host)
    data_dict = response["response"]
    if bool(data_dict):
        try:
            containers = str(data_dict["apps_containers"])
            time = str(data_dict["report_creation_time"])
            click.echo(
                click.style(
                    key + " at time:" + time + " containers:" + str(containers),
                    fg="blue",
                )
            )
            return {
                "error": False,
                "response": key
                + " at time:"
                + time
                + " containers:"
                + str(containers),
            }
        except Exception as e:
            click.echo(click.style(e, fg="red"))
            # sys.exit()
            raise ErrorHandling
    else:
        click.echo(click.style("No key matches {}".format(key), fg="red"))
        return {"error": True, "response": "no key matches {}".format(key)}

getIndividualVitals(device_group, host)

Fetches the vitals across device groups and host combinations

Parameters:

Name Type Description Default
device_group

The device group being queried

required

host : string The host being queried

TODO: REST-fy this function, currently executes sys.exit()

Inorder to make Cache rest friendly replaced sys.exit() with raising exceptions which will get excepted in getAssetsForAll(self,asset,device_group_id="all",host_id="all") and eventually return a dictionary there {"error": True, "response": reason for error}

Source code in src/Cache.py
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
def getIndividualVitals(self, device_group, host):
    """
    Fetches the vitals across device groups and host combinations
    Parameters
    ----------
    device_group: string
        The device group being queried

    host : string
        The host being queried

    TODO: REST-fy this function, currently executes sys.exit()

    Inorder to make Cache rest friendly replaced sys.exit() with raising exceptions which will get excepted
    in getAssetsForAll(self,asset,device_group_id="all",host_id="all") and eventually return a dictionary there
    {"error": True, "response": reason for error}

    """
    response, key = self.unpickleData(device_group, host)
    data_dict = response["response"]
    if bool(data_dict):
        try:
            mem = str(data_dict["memory_usage"])
            disk = str(data_dict["root_disk_usage"])
            cpu_core_use = str(data_dict["cpu_usage"]["cores"])
            cpu_pct_use = str(data_dict["cpu_usage"]["used_percent"])
            time = str(data_dict["report_creation_time"])
            click.echo(
                click.style(
                    key
                    + "at time:"
                    + time
                    + "\t mem:"
                    + mem
                    + "\t"
                    + "disk:"
                    + disk
                    + "\t"
                    + "cpu_cores:"
                    + cpu_core_use
                    + "\t"
                    + "cpu_percent:"
                    + cpu_pct_use,
                    fg="blue",
                )
            )
            return {
                "error": False,
                "response": key
                + "at time:"
                + time
                + "\t mem:"
                + mem
                + "\t"
                + "disk:"
                + disk
                + "\t"
                + "cpu_cores:"
                + cpu_core_use
                + "\t"
                + "cpu_percent:"
                + cpu_pct_use,
            }

        except Exception as e:
            click.echo(click.style(e, fg="red"))
            # sys.exit()
            raise ErrorHandling
    else:
        click.echo(click.style("No key matches {}".format(key), fg="red"))
        return {"error": True, "response": "no key matches {}".format(key)}

keyPartition(raw_key)

Partitions key based on the delimiter "_"

Parameters:

Name Type Description Default
raw_key string

Raw key to be partitioned

required

Returns:

Name Type Description
component_tuple list

A tuple of three key components

Source code in src/Cache.py
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
def keyPartition(self, raw_key):
    """
    Partitions key based on the delimiter "_"

    Parameters
    ----------
    raw_key : string
        Raw key to be partitioned

    Returns
    -------

    component_tuple : list
        A tuple of three key components

    """
    key = str(raw_key).replace("'", "")
    key_components = str(key).split("_")
    return key_components[0], key_components[1], key_components[2]

scanLatest()

Iteratively scan and find the latest matching key and return the most freshest key

Returns:

Name Type Description
host_dict dict

A dictionary containing the latest key value pairs

Source code in src/Cache.py
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
def scanLatest(
    self,
):

    """
    Iteratively scan and find the latest matching key and return the most freshest key

    Returns
    -------
    host_dict : dict
        A dictionary containing the latest key value pairs
    """

    host_dict = {}

    for key in self.redisObj.scan_iter(self.CACHE_PREFIX + "_*"):
        prefix, timestamp, host = self.keyPartition(key)
        if host not in host_dict.keys():
            host_dict[host] = timestamp
        else:
            if host_dict[host] < timestamp:
                host_dict[host] = timestamp
    return host_dict

unpickleData(device_group, host)

Given a device group and host, unpickles the query result (vitals and containers) from Redis cache.

Parameters:

Name Type Description Default
device_group

The device group being queried

required

host : string The host being queried

Returns:

Name Type Description
mapping_dict dict

A dict containing { "host_queried":the query results for the host, "device_group_queried":the query results for the device group, "response":a boolean value depending on success of query }

key : string The key that led to the match

Source code in src/Cache.py
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
def unpickleData(self, device_group, host):

    """
    Given a device group and host, unpickles the query result (vitals and containers) from Redis cache.

    Parameters
    ----------
    device_group: string
        The device group being queried

    host : string
        The host being queried

    Returns
    -------
    mapping_dict : dict
        A dict containing {
                            "host_queried":the query results for the host,
                            "device_group_queried":the query results for the device group,
                            "response":a boolean value depending on success of query
                          }

    key : string
        The key that led to the match

    """

    key = str(device_group + "@" + host)
    host_dict = self.scanLatest()
    # response = {"host_queried": host, "device_group_queried": device_group, "response": response}
    if key in host_dict.keys():
        timestamp = host_dict[key]
        try:
            dataObj = self.redisObj.get(
                self.CACHE_PREFIX + "_" + timestamp + "_" + key
            )
            data_dict = pickle.loads(dataObj)
            return {
                "host_queried": host,
                "device_group_queried": device_group,
                "response": data_dict,
            }, key
        except Exception as e:
            click.echo(click.style(e, fg="red"))
            return {
                "host_queried": host,
                "device_group_queried": device_group,
                "response": {},
            }, key
    else:
        click.echo(
            click.style("{} not found in cache reports".format(key), fg="red")
        )
        return {
            "host_queried": host,
            "device_group_queried": device_group,
            "response": {},
        }, key