DBus API to retrieve images data and vulnerability information

images now either print to tty or return json data via dbus

Make encoding and decoding work properly for both Python2 and Python3

Create Cockpit JavaScript test client that will call the DBus API
and receive information.

Closes: #494
Approved by: rhatdan

Atomic/atomic.py

@@ -687,17 +687,7 @@ class Atomic(object):
 
         return True
 
-    def images(self):
-        def split_repo_tags(_images):
-            sub_list = [item.split(":") for sublist in _images for item
-                         in sublist['RepoTags']]
-            repo_tags = []
-            for repo in sub_list:
-                if len(repo) > 2:
-                    repo = [repo[0] + repo[1], repo[2]]
-                repo_tags.append(repo)
-            return repo_tags
-
+    def display_all_image_info(self):
         def get_col_lengths(_images):
             '''
             Determine the max length of the repository and tag names
@@ -705,7 +695,7 @@ class Atomic(object):
             :return: a set with len of repository and tag
             If there are no images, return 1, 1
             '''
-            repo_tags = split_repo_tags(_images)
+            repo_tags = [[i["repo"], i["tag"]] for i in _images]
             # Integer additions below are for column padding
             # 7 == 1 for dangling, 2 for spacing, 4 for highlighting
             if repo_tags:
@@ -714,12 +704,9 @@ class Atomic(object):
             else:
                 return 1, 1
 
-        _images = self.get_images(get_all=self.args.all)
-
-        used_image_ids = [x['ImageID'] for x in self.get_containers()]
+        _images = self.images()
 
         if len(_images) >= 0:
-            vuln_ids = self.get_vulnerable_ids()
             _max_repo, _max_tag = get_col_lengths(_images)
             if self.args.truncate:
                 _max_id = 14
@@ -737,7 +724,42 @@ class Atomic(object):
                                               "CREATED",
                                               "VIRTUAL SIZE",
                                               "TYPE"))
+
             for image in _images:
+                if self.args.filter:
+                    image_info = {"repo" : image['repo'], "tag" : image['tag'], "id" : image['id'],
+                                  "created" : image['created'], "size" : image['virtual_size'], "type" : image['type']}
+                    if not self._filter_include_image(image_info):
+                        continue
+                if self.args.quiet:
+                    util.write_out(image['id'])
+
+                else:
+                    indicator = ""
+                    if image["is_dangling"]:
+                        indicator += "*"
+                    elif image["used_image"]:
+                        indicator += ">"
+                    if image["vulnerable"]:
+                        space = " " if len(indicator) < 1 else ""
+                        if util.is_python2:
+                            indicator = indicator + self.skull + space
+                        else:
+                            indicator = indicator + str(self.skull, "utf-8") + space
+                    util.write_out(col_out.format(indicator, image['repo'], image['tag'], image['id'], image['created'], image['virtual_size'], image['type']))
+            util.write_out("")
+            return
+
+    def images(self):
+        _images = self.get_images()
+        all_image_info = []
+
+        if len(_images) >= 0:
+            vuln_ids = self.get_vulnerable_ids()
+            all_vuln_info = json.loads(self.get_all_vulnerable_info())
+            used_image_ids = [x['ImageID'] for x in self.get_containers()]
+            for image in _images:
+                image_dict = dict()
                 repo, tag = image["RepoTags"][0].rsplit(":", 1)
                 if "Created" in image:
                     created = time.strftime("%F %H:%M", time.localtime(image["Created"]))
@@ -748,35 +770,24 @@ class Atomic(object):
                 else:
                     virtual_size = ""
 
-                if self.is_dangling(repo):
-                    indicator = "*"
-                elif image['Id'] in used_image_ids:
-                    indicator = ">"
-                else:
-                    indicator = ""
-
-                if image['Id'] in vuln_ids:
-                    space = " " if len(indicator) < 1 else ""
-                    indicator = indicator + self.skull + space
-
+                image_dict["is_dangling"] = self.is_dangling(repo)
+                image_dict["used_image"] = image["Id"] in used_image_ids
+                image_dict["vulnerable"] = image["Id"] in vuln_ids
                 image_id = image["Id"][:12] if self.args.truncate else image["Id"]
                 image_type = image['ImageType']
-                if self.args.filter:
-                    image_info = {"repo" : repo, "tag" : tag, "id" : image_id,
-                                  "created" : created, "size" : virtual_size, "type" : image_type}
-                    if not self._filter_include_image(image_info):
-                        continue
-
-                if self.args.quiet:
-                    util.write_out(image_id)
+                image_dict["repo"] = repo
+                image_dict["tag"] = tag
+                image_dict["id"] = image_id
+                image_dict["created"] = created
+                image_dict["virtual_size"] = virtual_size
+                image_dict["type"] = image_type
+                if image_dict["vulnerable"]:
+                    image_dict["vuln_info"] = all_vuln_info[image["Id"]]
                 else:
-                    util.write_out(col_out.format(indicator, repo,
-                                                  tag, image_id,
-                                                  created,
-                                                  virtual_size,
-                                                  image_type))
-            util.write_out("")
-            return
+                    image_dict["vuln_info"] = dict()
+
+                all_image_info.append(image_dict)
+            return all_image_info
 
     def _check_if_image_present(self):
         self.inspect = self._inspect_image()
@@ -1090,6 +1101,17 @@ class Atomic(object):
         if self.args.debug:
             self.debug = True
 
+    def get_all_vulnerable_info(self):
+        """
+        Will simply read and return the entire /var/lib/atomic/scan_summary.json
+        as a JSON string so it can then be parsed appropriately.
+        """
+        try:
+            return open(os.path.join(self.results, "scan_summary.json"), "r").read()
+        except IOError:
+            return "{}"
+
+
     def get_vulnerable_ids(self):
         """
         Reads in /var/lib/atomic/scan_summary.json and returns a list of all

Atomic/scan.py

@@ -146,6 +146,8 @@ class Scan(Atomic):
 
             # record environment
             self.record_environment()
+
+            self.write_persistent_data()
         finally:
             # unmount all the rootfs
             self._unmount_rootfs_in_dir()
@@ -196,6 +198,11 @@ class Scan(Atomic):
 
         return scan_list
 
+    def _get_roots_path_from_bind_name(self, in_bind_name):
+        for _path, bind_path in self.rootfs_mappings.items():
+            if bind_path == os.path.basename(os.path.split(in_bind_name)[0]):
+                return _path
+
     def get_scan_data(self):
         results = []
         json_files = self._get_json_files()
@@ -251,21 +258,12 @@ class Scan(Atomic):
         Write results of the scan to stdout
         :return: None
         """
-        def _get_roots_path_from_bind_name(in_bind_name):
-            for _path, bind_path in self.rootfs_mappings.items():
-                if bind_path == os.path.basename(os.path.split(in_bind_name)[0]):
-                    return _path
-
-        persistent_data = {}
         json_files = self._get_json_files()
         for json_file in json_files:
             json_results = json.load(open(json_file))
 
             uuid = os.path.basename(json_results['UUID']) if len(self.args.rootfs) == 0 \
-                else _get_roots_path_from_bind_name(json_file)
-
-            # Get data from the results for persistent use
-            persistent_data[uuid] = self.get_persist_data(json_results, json_file)
+                else self._get_roots_path_from_bind_name(json_file)
 
             name1 = uuid if len(self.args.rootfs) > 1 else self._get_input_name_for_id(uuid)
             if len(self.args.rootfs) == 0 and not self._is_iid(uuid):
@@ -305,9 +303,6 @@ class Scan(Atomic):
                                .format(' ' * 5, self._get_input_name_for_id(uuid)))
         util.write_out("\nFiles associated with this scan are in {}.\n".format(self.results_dir))
 
-        self.write_persistent_data(persistent_data)
-
-
     def _output_custom(self, value, indent):
         space = ' ' * indent
         next_indent = indent + 2
@@ -460,7 +455,15 @@ class Scan(Atomic):
         persist['json_file'] = json_file
         return persist
 
-    def write_persistent_data(self, new_data):
+
+    def write_persistent_data(self):
+        new_data = dict()
+        json_files = self._get_json_files()
+        for json_file in json_files:
+            json_results = json.load(open(json_file))
+            uuid = os.path.basename(json_results['UUID']) if len(self.args.rootfs) == 0 \
+                else self._get_roots_path_from_bind_name(json_file)
+            new_data[uuid] = self.get_persist_data(json_results, json_file)
         summary_file = os.path.join(self.results, "scan_summary.json")
         if not os.path.exists(summary_file):
             persistent_data = new_data
@@ -473,10 +476,9 @@ class Scan(Atomic):
                     persistent_data[uuid] = new_data[uuid]
 
             # Clean up old data
-            for uuid in persistent_data.keys():
+            for uuid in list(persistent_data):
                 if uuid not in iids and uuid not in cids:
                     del persistent_data[uuid]
 
         with open(summary_file, 'w') as f:
             json.dump(persistent_data, f, indent=4)
-

Atomic/util.py

@@ -144,7 +144,9 @@ def _output(fd, output, lf):
     fd.flush()
 
     if is_python2:
-        fd.write(output.encode('utf-8') + lf)
+        if isinstance(output, unicode): #pylint: disable=undefined-variable,unicode-builtin
+            output = output.encode('utf-8')
+        fd.write(output + lf)
     else:
         fd.write(output + str(lf))
 

atomic

@@ -317,7 +317,7 @@ def create_parser(help_text):
                                               help=_("list container images on your system"),
                                               epilog="atomic images by default will list all installed "
                                                      "container images on your system.")
-    list_parser.set_defaults(func='images')
+    list_parser.set_defaults(func='display_all_image_info')
 
     list_parser.add_argument("-a", "--all", dest="all", default=False,
                              action="store_true",

atomic_client.py

@@ -69,6 +69,14 @@ class AtomicDBus (object):
     def update(self, image):
         self.dbus_object.Update(image, dbus_interface="org.atomic", timeout = 2147400)
 
+    @polkit.enable_proxy
+    def images(self):
+        return self.dbus_object.Images(dbus_interface="org.atomic", timeout = 2147400)
+
+    @polkit.enable_proxy
+    def vulnerable(self):
+        return self.dbus_object.VulnerableInfo(dbus_interface="org.atomic", timeout = 2147400)
+
 #For outputting the list of scanners
 def print_scan_list(all_scanners):
     if len(all_scanners) == 0:
@@ -150,6 +158,12 @@ if __name__ == "__main__":
 
         elif(sys.argv[1] == "update"):
             dbus_proxy.update(sys.argv[2])
-            
+
+        elif(sys.argv[1] == "images"):
+            print(json.loads(dbus_proxy.images()))
+
+        elif(sys.argv[1] == "vulnerable"):
+            print(json.loads(dbus_proxy.vulnerable()))
+
     except dbus.DBusException as e:
         print (e)

atomic_dbus.py

@@ -43,6 +43,9 @@ class atomic_dbus(slip.dbus.service.Object):
             self.images = False
             self.containers = False
             self.container = False
+            self.prune = False
+            self.heading = False
+            self.truncate = False
 
     def __init__(self, *p, **k):
         super(atomic_dbus, self).__init__(*p, **k)
@@ -244,6 +247,23 @@ class atomic_dbus(slip.dbus.service.Object):
         self.atomic.set_args(args)
         self.atomic.update()
 
+    # The Images method will list all installed container images on the system.
+    @slip.dbus.polkit.require_auth("org.atomic.read")
+    @dbus.service.method("org.atomic", in_signature='', out_signature='s')
+    def Images(self):
+        args = self.Args()
+        self.atomic.set_args(args)
+        return json.dumps(self.atomic.images())
+
+    # The Vulnerable method will send back information that says
+    # whether or not an installed container image is vulnerable
+    @slip.dbus.polkit.require_auth("org.atomic.read")
+    @dbus.service.method("org.atomic", in_signature='', out_signature='s')
+    def VulnerableInfo(self):
+        args = self.Args()
+        self.atomic.set_args(args)
+        return self.atomic.get_all_vulnerable_info()
+
 
 if __name__ == "__main__":
     mainloop = GLib.MainLoop()

tests/atomic-client.js

@@ -0,0 +1,167 @@
+require([
+    "jquery",
+    "base1/cockpit",
+], function($, cockpit) {
+  var input = $("#new");
+  var service = cockpit.dbus("org.atomic");
+  var proxy = service.proxy("org.atomic", "/org/atomic/object");
+  proxy.wait(function () {
+    if (!proxy.valid) {
+        $('#ui').hide();
+        $('#curtain').show();
+    }
+    else {
+      run_scan_list();
+    }
+    $('body').show();
+  });
+
+  $("#Run").on("click", run_request);
+  function run_scan_list() {
+    var call = proxy.ScanList();
+    call.done(function(result) {
+      response = JSON.parse(result);
+      for (var i = 0; i < response.length; i++) {
+        var radio = document.createElement('input');
+        radio.type = "radio";
+        radio.setAttribute("name", "scanner");
+        radio.setAttribute("value", response[i]["scanner_name"]);
+        var label = document.createElement('label')
+        label.htmlFor = "id";
+        label.appendChild(document.createTextNode(response[i]["scanner_name"]));
+        document.body.appendChild(radio);
+        document.body.appendChild(label);
+        scanned_list = response[i]["scans"];
+        for (var j = 0; j < scanned_list.length; j++) {
+          var radio_type = document.createElement('input');
+          radio_type.type = "radio";
+          radio_type.setAttribute("name", "scan_type");
+          radio_type.setAttribute("value", scanned_list[j]["name"]);
+          label = document.createElement('label')
+          label.htmlFor = "id";
+          label.appendChild(document.createTextNode(scanned_list[j]["name"]));
+          document.body.appendChild(radio_type);
+          document.body.appendChild(label);
+        }
+      }
+      run_images();
+    });
+
+    call.fail(function(error) {
+      console.warn(error);
+    });
+  }
+
+  function run_request() {
+    var scan_targets = [];
+    $('input[name="image"]:checked').each(function() {
+      scan_targets.push($(this).val());
+    });
+    if(typeof scan_targets == "undefined") {
+      scan_targets = [];
+    }
+    var scanner = $('input[name="scanner"]:checked').val();
+    if(typeof scanner == "undefined") {
+      scanner = '';
+    }
+    var scan_type = $('input[name="scan_type"]:checked').val();
+    if(typeof scan_type == "undefined") {
+      scan_type = '';
+    }
+    run_scan(scan_targets, scanner, scan_type)
+  }
+
+  function run_scan_async(scan_targets, scanner, scan_type) {
+    var call = proxy.ScheduleScan(scan_targets, scanner, scan_type, rootfs, false, false, false);
+    call.done(function(result) {
+      while(true){
+        NewCall = proxy.GetScanResults(result);
+        NewCall.done(function(data) {
+          if(data.length > 0) {
+            console.log(data);
+          }
+        });
+      }
+    });
+
+    call.fail(function(error) {
+      console.warn(error);
+    });
+  }
+
+  function run_scan(scan_targets, scanner, scan_type) {
+    var call;
+    call = proxy.Scan(scan_targets, scanner, scan_type, [], false, false, false)
+    call.done(function(result) {
+      var label = document.createElement('label')
+      label.htmlFor = "id";
+      label.appendChild(document.createTextNode(JSON.stringify(result)));
+      document.body.appendChild(label);
+    });
+
+    call.fail(function(error) {
+      console.warn(error);
+    });
+  }
+
+  function run_vulnerable_info() {
+    var call = proxy.VulnerableInfo();
+    call.done(function(result) {
+      console.log(result);
+    });
+
+    call.fail(function(error) {
+      console.warn(error);
+    });
+  }
+
+  function run_update(image) {
+    var call = proxy.Update(image);
+    call.done(function() {
+      console.log("Success");
+    });
+
+    call.fail(function(error) {
+      console.warn(error);
+    });
+  }
+
+  function run_images() {
+    var call = proxy.Images();
+    var text = "Repository            Last Scanned\n";
+    call.done(function(result) {
+      response = JSON.parse(result);
+      for (var i = 2; i < response.length; i++) {
+        text += response[i]["repo"] + "            ";
+        var checkbox = document.createElement('input');
+        checkbox.type = "checkbox";
+        checkbox.setAttribute("name", "image");
+        checkbox.setAttribute("value", response[i]["repo"]);
+        var label = document.createElement('label')
+        label.htmlFor = "id";
+        label.appendChild(document.createTextNode(response[i]["repo"]));
+        document.body.appendChild(checkbox);
+        document.body.appendChild(label);
+        if ("Time" in response[i]["vuln_info"]) {
+          text += response[i]["vuln_info"]["Time"] + " ";
+        }
+
+        if(response[i]["vulnerable"]) {
+          text += "*\n";
+        }
+
+        else {
+          text += "\n";
+        }
+      }
+      label = document.createElement('label');
+      label.htmlFor = "id";
+      label.appendChild(document.createTextNode(text));
+      document.body.appendChild(label);
+    });
+
+    call.fail(function(error) {
+      console.warn(error);
+    });
+  }
+});

tests/atomicClient.html

@@ -4,6 +4,8 @@
     <link href="../base1/cockpit.css" type="text/css" rel="stylesheet">
     <script src="../base1/jquery.js"></script>
     <script src="../base1/cockpit.js"></script>
+    <script src="../base1/bundle.js"></script>
+    <script src = "atomic-client.js"></script>
 </head>
 <body hidden>
     <div id="ui" class="container-fluid" style='max-width: 400px'>
@@ -12,10 +14,6 @@
                 <td><label class="control-label">Atomic</label></td>
                 <td><span id="current"></span></td>
             </tr>
-            <tr>
-                <td><label class="control-label" for="new">Feature</label></td>
-                <td><input class="form-control" id="new" value=""></td>
-            </tr>
             <tr>
                 <td><button class="btn btn-default btn-primary" id="Run">Run</button></td>
 		<td><span id="failure"></span></td>
@@ -26,247 +24,5 @@
     <div id="curtain" hidden>
         Atomic D-Bus API not available.
     </div>
-
-    <script>
-            var input = $("#new");
-            var service = cockpit.dbus("org.atomic");
-            var proxy = service.proxy("org.atomic", "/org/atomic/object");
-
-            proxy.wait(function () {
-                if (!proxy.valid) {
-                    $('#ui').hide();
-                    $('#curtain').show();
-                }
-                $('body').show();
-            });
-
-            $("#Run").on("click", run_command);
-
-            function run_version(command) {
-              var call;
-              if(command[2] == "-r") {
-                call = proxy.Version([command[1]], true);
-              }
-
-              else {
-                call = proxy.Version([command[1]], false);
-              }
-
-              call.done(function(result) {
-                console.log(JSON.stringify(result));
-              });
-
-              call.fail(function(error) {
-                  console.warn(error);
-                });
-
-              }
-
-            function run_scan_list() {
-              var call = proxy.ScanList();
-              call.done(function(result) {
-                response = JSON.parse(result);
-                console.log(JSON.stringify(result));
-              });
-
-              call.fail(function(error) {
-                console.warn(error);
-              });
-
-            }
-
-            function run_storage_reset() {
-              var call = proxy.StorageReset();
-              call.done(function() {
-                console.log("Success!");
-              });
-
-              call.fail(function(error) {
-                console.warn(error);
-              });
-            }
-
-            function run_storage_import(command) {
-              var graph = "/var/lib/docker";
-              var import_location = "/var/lib/atomic/migrate";
-              if(command.indexOf("--graph") != -1) {
-                graph = command[command.indexOf("--graph") + 1];
-              }
-
-              if (command.indexOf("--import_location") != -1) {
-                import_location = command[command.indexOf("--import_location") + 1];
-              }
-
-              var call = proxy.StorageImport(graph, import_location);
-              call.done(function() {
-                  console.log("Success!");
-                });
-
-                call.fail(function(error) {
-                  console.warn(error);
-                });
-              }
-
-            function run_storage_export(command) {
-              var graph = "/var/lib/docker";
-              var import_location = "/var/lib/atomic/migrate";
-              var force = false;
-              if(command.indexOf("--graph") != -1) {
-                graph = command[command.indexOf("--graph") + 1];
-                console.log(graph);
-              }
-
-              if (command.indexOf("--import_location") != -1) {
-                import_location = command[command.indexOf("--inport_location") + 1];
-                console.log(import_location);
-              }
-
-              if(command.indexOf("--force") != -1) {
-                force = true;
-              }
-
-              var call = proxy.StorageExport(graph, import_location, force);
-              call.done(function() {
-                console.log("Success!");
-              });
-
-              call.fail(function(error) {
-                console.warn(error);
-              });
-            }
-
-            function run_scan_async(command) {
-              var scan_targets = [];
-              var scanner = '';
-              var scan_type = '';
-              var rootfs = [];
-              var call;
-              if(command.indexOf("--all") != -1) {
-                call = proxy.ScheduleScan(scan_targets, scanner, scan_type, rootfs, true, false, false);
-              }
-
-              else if(command.indexOf("--images") != -1) {
-                call = proxy.ScheduleScan(scan_targets, scanner, scan_type, rootfs, false, true, false);
-              }
-
-              else if(command.indexOf("--containers") != -1) {
-                call = proxy.ScheduleScan(scan_targets, scanner, scan_type, rootfs, false, false, true);
-              }
-
-              else {
-                call = proxy.ScheduleScan([command[1]], scanner, scan_type, rootfs, false, false, false);
-              }
-              call.done(function(result) {
-                while(true){
-                  NewCall = proxy.GetScanResults(result);
-                  NewCall.done(function(data) {
-                    if(data.length > 0) {
-                      console.log(data);
-                    }
-                  });
-                }
-              });
-
-              call.fail(function(error) {
-                console.warn(error);
-              });
-
-            }
-
-            function run_scan(command) {
-              var scan_targets = [];
-              var scanner = '';
-              var scan_type = '';
-              var rootfs = [];
-              var call;
-              if(command.indexOf("--all") == 1) {
-                call = proxy.Scan(scan_targets, scanner, scan_type, rootfs, true, false, false);
-              }
-
-              else if(command.indexOf("--images") == 1) {
-                call = proxy.Scan(scan_targets, scanner, scan_type, rootfs, false, true, false);
-              }
-
-              else if(command.indexOf("--containers") == 1) {
-                call = proxy.Scan(scan_targets, scanner, scan_type, rootfs, false, false, true);
-              }
-
-              else {
-                call = proxy.Scan([command[1]], scanner, scan_type, rootfs, false, false, false);
-              }
-
-              call.done(function(result) {
-                console.log(JSON.stringify(result));
-              });
-
-              call.fail(function(error) {
-                console.warn(error);
-              });
-            }
-
-            function run_diff(command) {
-              var call = proxy.Diff(command[1], command[2]);
-              call.done(function(result) {
-                console.log(JSON.stringify(result));
-              });
-
-              call.fail(function(error) {
-                console.warn(error);
-              });
-            }
-
-            function run_verify(command) {
-              var call = proxy.Verify([command[1]]);
-              call.done(function(result) {
-                console.log(JSON.stringify(result));
-              });
-
-              call.fail(function(error) {
-                console.warn(error);
-              });
-            }
-
-            //Depending on what is entered, it will send the command to the appropriate function to be parsed.
-            function run_command() {
-              var command = (input.val()).split(" ");
-              var call;
-              if(command[0].toLowerCase() == "version") {
-                run_version(command);
-              }
-
-              else if(command[0].toLowerCase() == "scan") {
-                if(command.indexOf("--list") != -1) {
-                  run_scan_list();
-                }
-                else {
-                  run_scan(command);
-                }
-              }
-
-              else if(command[0].toLowerCase() == "storage") {
-                if(command.indexOf("--reset") != -1) {
-                  run_storage_reset()
-                }
-
-                else if(command.indexOf("--import") != -1) {
-                  run_storage_import(command)
-                }
-
-                else if(command.indexOf("--export") != -1) {
-                  run_storage_export(command)
-                }
-
-              }
-
-              else if(command[0].toLowerCase() == "diff") {
-                run_diff(command)
-              }
-
-              else if(command[0].toLowerCase() == "verify") {
-                run_verify(command)
-              }
-            }
-
-    </script>
 </body>
 </html>

tests/manifest.json

@@ -6,7 +6,5 @@
             "label": "Atomic",
             "path": "atomicClient.html"
         }
-    },
-
-    "content-security-policy": "default-src 'self' 'unsafe-inline' 'unsafe-eval'"
+    }
 }