Merge branch 'master' into yixue_branch

Author: lemduc

Bug_Extractor.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 132,
+   "execution_count": 23,
    "metadata": {
     "collapsed": false
    },
@@ -12,7 +12,7 @@
      "output_type": "stream",
      "text": [
       "Hadoop Common\n",
-      "......................................................................Total number of issues: 6822\n"
+      ".Total number of issues: 5\n"
      ]
     }
    ],
@@ -62,7 +62,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 133,
+   "execution_count": 25,
    "metadata": {
     "collapsed": false
    },
@@ -71,43 +71,49 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "......................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................"
+      "....."
      ]
     }
    ],
    "source": [
     "import time\n",
     "import datetime\n",
     "from datetime import datetime, date\n",
+    "from collections import OrderedDict\n",
     "\n",
     "storeIssues = []\n",
     "\n",
     "timeFormat = \"%Y-%m-%dT%H:%M:%S.000+0000\"\n",
     "\n",
     "for issue in issues:\n",
     "    print('.', end=\"\")\n",
-    "    exportedData = []\n",
+    "    exportedData = OrderedDict([])\n",
     "    \n",
     "    try:\n",
-    "        affectversion = issue.fields.versions[0].name\n",
+    "        affectversion = {'affect:':issue.fields.versions[0].name}\n",
     "    except Exception:\n",
-    "        affectversion = \"\"\n",
+    "        affectversion = {'affect:':\"\"}\n",
     "    \n",
     "    try:\n",
-    "        fixversion    = issue.fields.fixVersions[0].name\n",
+    "        fixversion    = {'fix:':issue.fields.fixVersions[0].name}\n",
     "    except Exception:\n",
-    "        fixversion = \"\"\n",
+    "        fixversion = {'fix:':\"\"}\n",
     "        \n",
-    "    priority      = issue.fields.priority\n",
+    "    priority      = {'priority':issue.fields.priority.name}\n",
     "    resolvedDate  = datetime.strptime(issue.fields.resolutiondate, timeFormat)\n",
     "    createdDate   = datetime.strptime(issue.fields.created, timeFormat)\n",
-    "    fixdays       = resolvedDate - createdDate\n",
+    "    fixdays       = {'time':(resolvedDate - createdDate).seconds}\n",
+    "    issue_type    = {'type':issue.fields.issuetype.name}\n",
     "   \n",
     "    #print(affectversion)\n",
     "    #print(fixversion)\n",
     "    #print(priority)\n",
     "    #print(fixdays)\n",
-    "    exportedData.extend([affectversion, fixversion, priority.name, fixdays.seconds])\n",
+    "    exportedData.update(affectversion)\n",
+    "    exportedData.update(fixversion)\n",
+    "    exportedData.update(priority)\n",
+    "    exportedData.update(issue_type)\n",
+    "    exportedData.update(fixdays)\n",
     "    \n",
     "    DEV_STATUS = 'https://issues.apache.org/jira/rest/dev-status/1.0'\n",
     "    _issue = 'issue/detail?issueId=%s' % issue.id\n",
@@ -124,15 +130,17 @@
     "    except IndexError:\n",
     "        hasCommit = False\n",
     "    if hasCommit:\n",
-    "        for req in commits:\n",
+    "        commitList = []\n",
+    "        for commit in commits:\n",
     "            #print(req)\n",
     "            #print(issue.id) \n",
     "            patches = []\n",
     "            #print('%s\\n%s\\n\\n' % (req['displayId'], req['files']))\n",
-    "            for file in req['files']:\n",
-    "                patches.append(file['path'])\n",
-    "            exportedData.append(patches)\n",
-    "            #print(patches)    \n",
+    "            for file in commit['files']:\n",
+    "                patches.append({'filename':file['path']})\n",
+    "            commitList.append({'files':patches})\n",
+    "            #print(patches)\n",
+    "        exportedData.update({'commits':commitList})\n",
     "        storeIssues.append(exportedData)\n",
     "    \n",
     "\n",

Hadoop Common_data.json

@@ -0,0 +1,69 @@
+component, affected class, total class, percentages
+org.apache.hadoop.hdfs.server.namenode.metrics,2,2,100.0
+org.apache.hadoop.metrics.ganglia,1,2,50.0
+org.apache.hadoop.io.serializer,2,14,14.285714
+org.apache.hadoop.metrics2,7,17,41.17647
+org.apache.hadoop.log.metrics,1,3,33.333332
+org.apache.hadoop.security,7,42,16.666666
+org.apache.hadoop.fs.s3,1,17,5.882353
+org.apache.hadoop.metrics2.impl,4,31,12.903226
+org.apache.hadoop.mapreduce,9,35,25.714285
+org.apache.hadoop.io.file.tfile,10,56,17.857143
+org.apache.hadoop.hdfs.protocol,16,29,55.172413
+org.apache.hadoop.io.nativeio,3,5,60.0
+org.apache.hadoop.fs.shell,1,4,25.0
+org.apache.hadoop.security.authentication.client,1,8,12.5
+org.apache.hadoop.hdfs,6,32,18.75
+org.apache.hadoop.fs.ftp,3,4,75.0
+org.apache.hadoop.security.token,3,9,33.333332
+org.apache.hadoop.io,16,97,16.494844
+org.apache.hadoop.metrics2.source,1,1,100.0
+org.apache.hadoop.fs.kfs,2,5,40.0
+org.apache.hadoop.mapred,88,435,20.229885
+org.apache.hadoop.hdfs.server.protocol,6,26,23.076923
+org.apache.hadoop.io.compress,3,21,14.285714
+org.apache.hadoop.mapred.pipes,1,17,5.882353
+org.apache.hadoop.hdfs.security.token.delegation,3,7,42.857143
+org.apache.hadoop.util.hash,2,3,66.666664
+org.apache.hadoop.security.authentication.server,1,9,11.111111
+org.apache.hadoop.filecache,1,11,9.090909
+org.apache.hadoop.mapreduce.split,2,6,33.333332
+org.apache.hadoop.net,5,24,20.833334
+org.apache.hadoop.metrics2.sink.ganglia,1,8,12.5
+org.apache.hadoop.ipc,7,30,23.333334
+org.apache.hadoop.util.bloom,3,8,37.5
+org.apache.hadoop.io.compress.zlib,3,11,27.272728
+org.apache.hadoop.record.meta,1,8,12.5
+org.apache.hadoop.metrics2.lib,1,12,8.333333
+org.apache.hadoop.jmx,1,2,50.0
+org.apache.hadoop.mapreduce.lib.input,3,10,30.0
+org.apache.hadoop.metrics.spi,4,14,28.571428
+org.apache.hadoop.mapreduce.security.token.delegation,1,3,33.333332
+org.apache.hadoop.mapred.lib,2,52,3.8461537
+org.apache.hadoop.hdfs.security.token.block,1,9,11.111111
+org.apache.hadoop.metrics,3,8,37.5
+org.apache.hadoop.metrics.jvm,1,2,50.0
+org.apache.hadoop.metrics2.util,2,11,18.181818
+org.apache.hadoop.log,2,3,66.666664
+org.apache.hadoop.fs.s3native,1,7,14.285714
+org.apache.hadoop.mapred.join,1,33,3.030303
+org.apache.hadoop.http,4,10,40.0
+org.apache.hadoop.hdfs.server.datanode.metrics,1,2,50.0
+org.apache.hadoop.security.token.delegation,1,7,14.285714
+org.apache.hadoop.hdfs.web.resources,22,45,48.88889
+org.apache.hadoop.mapred.jobcontrol,2,2,100.0
+org.apache.hadoop.util,11,59,18.644068
+org.apache.hadoop.record.compiler.generated,5,7,71.42857
+org.apache.hadoop.mapred.lib.aggregate,1,18,5.5555553
+org.apache.hadoop.security.authorize,3,9,33.333332
+org.apache.hadoop.io.compress.bzip2,5,8,62.5
+org.apache.hadoop.hdfs.server.namenode,23,119,19.32773
+org.apache.hadoop.security.authentication.util,1,2,50.0
+org.apache.hadoop.fs.permission,2,11,18.181818
+org.apache.hadoop.hdfs.util,2,8,25.0
+org.apache.hadoop.hdfs.server.datanode,9,53,16.981133
+org.apache.hadoop.record,4,22,18.181818
+org.apache.hadoop.http.lib,1,4,25.0
+org.apache.hadoop.fs,23,90,25.555555
+org.apache.hadoop.hdfs.server.common,5,22,22.727272
+org.apache.hadoop.hdfs.server.balancer,4,12,33.333332