From 7c13d73601cd1a98b7815c325e62017db27cf79e Mon Sep 17 00:00:00 2001
From: bert <1131482051@qq.com>
Date: Tue, 3 Dec 2019 09:07:38 +0800
Subject: [PATCH 1/3] =?UTF-8?q?=E5=9B=BE=E7=89=87=E6=95=B0=E6=8D=AE?=
 =?UTF-8?q?=E9=9B=86=E5=A4=84=E7=90=86?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

删除不必要信息，整合保留图片地址、图片标签和图片内的详细信息
---
 datasetHandleCode/datasetHandle.ipynb | 580 ++++++++++++++++++++++++++
 1 file changed, 580 insertions(+)
 create mode 100644 datasetHandleCode/datasetHandle.ipynb

diff --git a/datasetHandleCode/datasetHandle.ipynb b/datasetHandleCode/datasetHandle.ipynb
new file mode 100644
index 0000000..d4cede5
--- /dev/null
+++ b/datasetHandleCode/datasetHandle.ipynb
@@ -0,0 +1,580 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = pd.read_csv('./train-annotations-bbox.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ImageID</th>\n",
+       "      <th>Source</th>\n",
+       "      <th>LabelName</th>\n",
+       "      <th>Confidence</th>\n",
+       "      <th>XMin</th>\n",
+       "      <th>XMax</th>\n",
+       "      <th>YMin</th>\n",
+       "      <th>YMax</th>\n",
+       "      <th>IsOccluded</th>\n",
+       "      <th>IsTruncated</th>\n",
+       "      <th>IsGroupOf</th>\n",
+       "      <th>IsDepiction</th>\n",
+       "      <th>IsInside</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>000002b66c9c498e</td>\n",
+       "      <td>xclick</td>\n",
+       "      <td>/m/01g317</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.012500</td>\n",
+       "      <td>0.195312</td>\n",
+       "      <td>0.148438</td>\n",
+       "      <td>0.587500</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>000002b66c9c498e</td>\n",
+       "      <td>xclick</td>\n",
+       "      <td>/m/01g317</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.025000</td>\n",
+       "      <td>0.276563</td>\n",
+       "      <td>0.714063</td>\n",
+       "      <td>0.948438</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>000002b66c9c498e</td>\n",
+       "      <td>xclick</td>\n",
+       "      <td>/m/01g317</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.151562</td>\n",
+       "      <td>0.310937</td>\n",
+       "      <td>0.198437</td>\n",
+       "      <td>0.590625</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>000002b66c9c498e</td>\n",
+       "      <td>xclick</td>\n",
+       "      <td>/m/01g317</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.256250</td>\n",
+       "      <td>0.429688</td>\n",
+       "      <td>0.651563</td>\n",
+       "      <td>0.925000</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>000002b66c9c498e</td>\n",
+       "      <td>xclick</td>\n",
+       "      <td>/m/01g317</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.257812</td>\n",
+       "      <td>0.346875</td>\n",
+       "      <td>0.235938</td>\n",
+       "      <td>0.385938</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "            ImageID  Source  LabelName  Confidence      XMin      XMax  \\\n",
+       "0  000002b66c9c498e  xclick  /m/01g317           1  0.012500  0.195312   \n",
+       "1  000002b66c9c498e  xclick  /m/01g317           1  0.025000  0.276563   \n",
+       "2  000002b66c9c498e  xclick  /m/01g317           1  0.151562  0.310937   \n",
+       "3  000002b66c9c498e  xclick  /m/01g317           1  0.256250  0.429688   \n",
+       "4  000002b66c9c498e  xclick  /m/01g317           1  0.257812  0.346875   \n",
+       "\n",
+       "       YMin      YMax  IsOccluded  IsTruncated  IsGroupOf  IsDepiction  \\\n",
+       "0  0.148438  0.587500           0            1          0            0   \n",
+       "1  0.714063  0.948438           0            1          0            0   \n",
+       "2  0.198437  0.590625           1            0          0            0   \n",
+       "3  0.651563  0.925000           1            0          0            0   \n",
+       "4  0.235938  0.385938           1            0          0            0   \n",
+       "\n",
+       "   IsInside  \n",
+       "0         0  \n",
+       "1         0  \n",
+       "2         0  \n",
+       "3         0  \n",
+       "4         0  "
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# dataset的数据为每张图像中每个物体的具体位置\n",
+    "dataset.head(5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1743042"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(dataset['ImageID'].unique())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset1 = pd.read_csv('./train-images-boxable-with-rotation.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1743042"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(dataset1['ImageID'].unique())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ImageID</th>\n",
+       "      <th>Subset</th>\n",
+       "      <th>OriginalURL</th>\n",
+       "      <th>OriginalLandingURL</th>\n",
+       "      <th>License</th>\n",
+       "      <th>AuthorProfileURL</th>\n",
+       "      <th>Author</th>\n",
+       "      <th>Title</th>\n",
+       "      <th>OriginalSize</th>\n",
+       "      <th>OriginalMD5</th>\n",
+       "      <th>Thumbnail300KURL</th>\n",
+       "      <th>Rotation</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>4fa8054781a4c382</td>\n",
+       "      <td>train</td>\n",
+       "      <td>https://farm3.staticflickr.com/5310/5898076654...</td>\n",
+       "      <td>https://www.flickr.com/photos/michael-beat/589...</td>\n",
+       "      <td>https://creativecommons.org/licenses/by/2.0/</td>\n",
+       "      <td>https://www.flickr.com/people/michael-beat/</td>\n",
+       "      <td>Michael Beat</td>\n",
+       "      <td>...die FNF-Kerze</td>\n",
+       "      <td>4405052</td>\n",
+       "      <td>KFukvivpCM5QXl5SqKe41g==</td>\n",
+       "      <td>https://c1.staticflickr.com/6/5310/5898076654_...</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>b37f763ae67d0888</td>\n",
+       "      <td>train</td>\n",
+       "      <td>https://c1.staticflickr.com/1/67/197493648_628...</td>\n",
+       "      <td>https://www.flickr.com/photos/drstarbuck/19749...</td>\n",
+       "      <td>https://creativecommons.org/licenses/by/2.0/</td>\n",
+       "      <td>https://www.flickr.com/people/drstarbuck/</td>\n",
+       "      <td>Karen</td>\n",
+       "      <td>Three boys on a hill</td>\n",
+       "      <td>494555</td>\n",
+       "      <td>9IzEn38GRNsVpATuv7gzEA==</td>\n",
+       "      <td>https://c3.staticflickr.com/1/67/197493648_628...</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>7e8584b0f487cb9e</td>\n",
+       "      <td>train</td>\n",
+       "      <td>https://c7.staticflickr.com/8/7056/7143870979_...</td>\n",
+       "      <td>https://www.flickr.com/photos/circasassy/71438...</td>\n",
+       "      <td>https://creativecommons.org/licenses/by/2.0/</td>\n",
+       "      <td>https://www.flickr.com/people/circasassy/</td>\n",
+       "      <td>CircaSassy</td>\n",
+       "      <td>A Christmas carol and The cricket on the heart...</td>\n",
+       "      <td>2371584</td>\n",
+       "      <td>3hQwu0iSzY1VIoXiwp0/Mg==</td>\n",
+       "      <td>https://c7.staticflickr.com/8/7056/7143870979_...</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>86638230febe21c4</td>\n",
+       "      <td>train</td>\n",
+       "      <td>https://farm5.staticflickr.com/5128/5301868579...</td>\n",
+       "      <td>https://www.flickr.com/photos/ajcreencia/53018...</td>\n",
+       "      <td>https://creativecommons.org/licenses/by/2.0/</td>\n",
+       "      <td>https://www.flickr.com/people/ajcreencia/</td>\n",
+       "      <td>Alex</td>\n",
+       "      <td>Abbey and Kenny</td>\n",
+       "      <td>949267</td>\n",
+       "      <td>onB+rCZnGQg5PRX7xOs18Q==</td>\n",
+       "      <td>https://c4.staticflickr.com/6/5128/5301868579_...</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>249086e72671397d</td>\n",
+       "      <td>train</td>\n",
+       "      <td>https://c6.staticflickr.com/4/3930/15342460029...</td>\n",
+       "      <td>https://www.flickr.com/photos/codnewsroom/1534...</td>\n",
+       "      <td>https://creativecommons.org/licenses/by/2.0/</td>\n",
+       "      <td>https://www.flickr.com/people/codnewsroom/</td>\n",
+       "      <td>COD Newsroom</td>\n",
+       "      <td>Suburban Law Enforcement Academy 20th Annivers...</td>\n",
+       "      <td>6541758</td>\n",
+       "      <td>MjpaAVbMAWbCusSaxI1D7w==</td>\n",
+       "      <td>https://c1.staticflickr.com/4/3930/15342460029...</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "            ImageID Subset                                        OriginalURL  \\\n",
+       "0  4fa8054781a4c382  train  https://farm3.staticflickr.com/5310/5898076654...   \n",
+       "1  b37f763ae67d0888  train  https://c1.staticflickr.com/1/67/197493648_628...   \n",
+       "2  7e8584b0f487cb9e  train  https://c7.staticflickr.com/8/7056/7143870979_...   \n",
+       "3  86638230febe21c4  train  https://farm5.staticflickr.com/5128/5301868579...   \n",
+       "4  249086e72671397d  train  https://c6.staticflickr.com/4/3930/15342460029...   \n",
+       "\n",
+       "                                  OriginalLandingURL  \\\n",
+       "0  https://www.flickr.com/photos/michael-beat/589...   \n",
+       "1  https://www.flickr.com/photos/drstarbuck/19749...   \n",
+       "2  https://www.flickr.com/photos/circasassy/71438...   \n",
+       "3  https://www.flickr.com/photos/ajcreencia/53018...   \n",
+       "4  https://www.flickr.com/photos/codnewsroom/1534...   \n",
+       "\n",
+       "                                        License  \\\n",
+       "0  https://creativecommons.org/licenses/by/2.0/   \n",
+       "1  https://creativecommons.org/licenses/by/2.0/   \n",
+       "2  https://creativecommons.org/licenses/by/2.0/   \n",
+       "3  https://creativecommons.org/licenses/by/2.0/   \n",
+       "4  https://creativecommons.org/licenses/by/2.0/   \n",
+       "\n",
+       "                              AuthorProfileURL        Author  \\\n",
+       "0  https://www.flickr.com/people/michael-beat/  Michael Beat   \n",
+       "1    https://www.flickr.com/people/drstarbuck/         Karen   \n",
+       "2    https://www.flickr.com/people/circasassy/    CircaSassy   \n",
+       "3    https://www.flickr.com/people/ajcreencia/          Alex   \n",
+       "4   https://www.flickr.com/people/codnewsroom/  COD Newsroom   \n",
+       "\n",
+       "                                               Title  OriginalSize  \\\n",
+       "0                                   ...die FNF-Kerze       4405052   \n",
+       "1                               Three boys on a hill        494555   \n",
+       "2  A Christmas carol and The cricket on the heart...       2371584   \n",
+       "3                                    Abbey and Kenny        949267   \n",
+       "4  Suburban Law Enforcement Academy 20th Annivers...       6541758   \n",
+       "\n",
+       "                OriginalMD5  \\\n",
+       "0  KFukvivpCM5QXl5SqKe41g==   \n",
+       "1  9IzEn38GRNsVpATuv7gzEA==   \n",
+       "2  3hQwu0iSzY1VIoXiwp0/Mg==   \n",
+       "3  onB+rCZnGQg5PRX7xOs18Q==   \n",
+       "4  MjpaAVbMAWbCusSaxI1D7w==   \n",
+       "\n",
+       "                                    Thumbnail300KURL  Rotation  \n",
+       "0  https://c1.staticflickr.com/6/5310/5898076654_...       0.0  \n",
+       "1  https://c3.staticflickr.com/1/67/197493648_628...       0.0  \n",
+       "2  https://c7.staticflickr.com/8/7056/7143870979_...       0.0  \n",
+       "3  https://c4.staticflickr.com/6/5128/5301868579_...       NaN  \n",
+       "4  https://c1.staticflickr.com/4/3930/15342460029...       0.0  "
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# dataset1的数据集为图片集的来源\n",
+    "dataset1.head(5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 提取dataset1中的图片ID和url\n",
+    "dataset2 = dataset1.drop(['Subset','OriginalURL','License','AuthorProfileURL','Author','Title','OriginalSize','OriginalMD5','Thumbnail300KURL','Rotation'],axis=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>ImageID</th>\n",
+       "      <th>OriginalLandingURL</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>4fa8054781a4c382</td>\n",
+       "      <td>https://www.flickr.com/photos/michael-beat/589...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>b37f763ae67d0888</td>\n",
+       "      <td>https://www.flickr.com/photos/drstarbuck/19749...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "            ImageID                                 OriginalLandingURL\n",
+       "0  4fa8054781a4c382  https://www.flickr.com/photos/michael-beat/589...\n",
+       "1  b37f763ae67d0888  https://www.flickr.com/photos/drstarbuck/19749..."
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset2.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save\n",
+    "dataset2.to_csv('./urllink.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['ImageID', 'OriginalLandingURL'], dtype='object')"
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset2.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset3 = dataset.drop(['Source','Confidence'],axis=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['ImageID', 'LabelName', 'XMin', 'XMax', 'YMin', 'YMax', 'IsOccluded',\n",
+       "       'IsTruncated', 'IsGroupOf', 'IsDepiction', 'IsInside'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset3.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save\n",
+    "dataset3.to_csv('./detail.csv',header=False) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
\ No newline at end of file

From 15e65698a0fd8614f5adc135f081470cb0ebadae Mon Sep 17 00:00:00 2001
From: bert <1131482051@qq.com>
Date: Wed, 4 Dec 2019 14:00:06 +0800
Subject: [PATCH 2/3] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=B8=8B=E8=BD=BD?=
 =?UTF-8?q?=E9=83=A8=E5=88=86=E7=9A=84=E4=BB=A3=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../ImagePoolProject/DatabaseQuery.go         | 18 +++++
 imagePoolCode/ImagePoolProject/Downloader.go  | 68 +++++++++++++++++++
 .../ImagePoolProject/conf/config.json         |  3 +
 imagePoolCode/ImagePoolProject/filecontorl.go | 42 ++++++++++++
 imagePoolCode/ImagePoolProject/go.mod         |  9 +++
 imagePoolCode/ImagePoolProject/go.sum         | 39 +++++++++++
 imagePoolCode/ImagePoolProject/main.go        | 24 +++++++
 7 files changed, 203 insertions(+)
 create mode 100644 imagePoolCode/ImagePoolProject/DatabaseQuery.go
 create mode 100644 imagePoolCode/ImagePoolProject/Downloader.go
 create mode 100644 imagePoolCode/ImagePoolProject/conf/config.json
 create mode 100644 imagePoolCode/ImagePoolProject/filecontorl.go
 create mode 100644 imagePoolCode/ImagePoolProject/go.mod
 create mode 100644 imagePoolCode/ImagePoolProject/go.sum
 create mode 100644 imagePoolCode/ImagePoolProject/main.go

diff --git a/imagePoolCode/ImagePoolProject/DatabaseQuery.go b/imagePoolCode/ImagePoolProject/DatabaseQuery.go
new file mode 100644
index 0000000..dbfcfb4
--- /dev/null
+++ b/imagePoolCode/ImagePoolProject/DatabaseQuery.go
@@ -0,0 +1,18 @@
+package main
+import (
+	"fmt"
+	"github.com/astaxie/beego/orm"
+)
+
+func init()  {
+}
+
+func GroupUrlGet(Group string){
+	o:=orm.NewOrm()
+	var obj []interface{}
+	_,err:=o.Raw("select imageID from imageLabelTable where labelName = ? and confidence=1 limit 10;",Group).QueryRows(&obj)
+	if err != nil {
+		fmt.Println(err)
+	}
+	fmt.Printf("%#v",obj)
+}
\ No newline at end of file
diff --git a/imagePoolCode/ImagePoolProject/Downloader.go b/imagePoolCode/ImagePoolProject/Downloader.go
new file mode 100644
index 0000000..42fa542
--- /dev/null
+++ b/imagePoolCode/ImagePoolProject/Downloader.go
@@ -0,0 +1,68 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"github.com/astaxie/beego/httplib"
+	"strconv"
+	"strings"
+	"sync"
+)
+//图片下载函数，保存图片到TempImage文件夹下
+func ImageDownload(imageID string ,url string )(status bool,err error){
+	resp,err := http.Get(url)
+	if err != nil {
+		fmt.Println(err)
+		return false,errors.New("can't connected")
+	}
+	if resp.StatusCode==200{
+		img,err:=os.OpenFile("./TempImage/"+imageID+".jpg",os.O_WRONLY|os.O_CREATE|os.O_EXCL,0666)
+		defer img.Close()
+		if err != nil {
+			println(err)
+		}
+		image:=make([]byte,10240)
+		for {
+			n,err := resp.Body.Read(image)
+			if err == io.EOF {
+				break
+			}
+			n,err = img.Write(image[:n])
+			if err != nil {
+				fmt.Println("img write:",err)
+			}
+		}
+		return true ,nil
+	} else{
+		err:=errors.New("StatusCode Error")
+		return false,err
+	}
+}
+func urlDownload(url string)[]string{
+
+	resp:=httplib.Get(url)
+	text,err:=resp.String()
+	if err != nil {
+		fmt.Println(err)
+	}
+	urls:=strings.Split(text,"\n")
+	return urls
+}
+func ImageNetDownload(urls []string){
+	println(len(urls))
+	w:=sync.WaitGroup{}
+
+	for i:=0;i<len(urls);i++{
+		w.Add(1)
+		println(i)
+		go func(i int,url string) {
+			url=strings.ReplaceAll(url,"\r","")
+			ImageDownload(strconv.Itoa(i),url)
+			w.Done()
+		}(i,urls[i])
+	}
+	w.Wait()
+}
\ No newline at end of file
diff --git a/imagePoolCode/ImagePoolProject/conf/config.json b/imagePoolCode/ImagePoolProject/conf/config.json
new file mode 100644
index 0000000..e6653c4
--- /dev/null
+++ b/imagePoolCode/ImagePoolProject/conf/config.json
@@ -0,0 +1,3 @@
+{
+  "size":
+}
\ No newline at end of file
diff --git a/imagePoolCode/ImagePoolProject/filecontorl.go b/imagePoolCode/ImagePoolProject/filecontorl.go
new file mode 100644
index 0000000..3c427f1
--- /dev/null
+++ b/imagePoolCode/ImagePoolProject/filecontorl.go
@@ -0,0 +1,42 @@
+package main
+
+
+import (
+	"github.com/astaxie/beego/httplib"
+	"errors"
+)
+var Url ="http://127.0.0.1:8080/group1"
+func init(){}
+
+func FileUploadToDFS(imageID string,group string) (md5 string,err error){
+	obj:=make(map[string]string,10)
+	req:=httplib.Post(Url+"/upload")
+	req.PostFile("file","./TempImage/"+imageID+".jpg")//注意不是全路径
+	req.Param("output","json")
+	req.Param("scene",group)
+	req.Param("path","")
+	req.ToJSON(&obj)
+	return obj["md5"] ,nil
+}
+func FileDeleteFromDFS(md5 string,group string)(bool,error){
+	var obj interface{}
+	req:=httplib.Get(Url+"/delete?md5="+md5)
+	req.ToJSON(&obj)
+	stat:=obj.(map[string]interface{})["status"].(string)
+	if stat=="ok"{
+		return true ,nil
+	}else if stat=="false"{
+		return false ,	errors.New("File Already Delete!")
+	}
+	return false,nil
+}
+func FileStat()(num int,size int){
+	var obj interface{}
+	mp :=make(map[string]int,10)
+	mp["a"]=1
+	req:=httplib.Get("http://127.0.0.1:8080/group1/stat")
+	req.ToJSON(&obj)
+	fileNum:=int(obj.(map[string]interface{})["data"].([]interface{})[0].(map[string]interface{})["fileCount"].(float64))
+	fileSize:=int(obj.(map[string]interface{})["data"].([]interface{})[0].(map[string]interface{})["totalSize"].(float64))
+	return fileNum,fileSize
+}
diff --git a/imagePoolCode/ImagePoolProject/go.mod b/imagePoolCode/ImagePoolProject/go.mod
new file mode 100644
index 0000000..05fa2cd
--- /dev/null
+++ b/imagePoolCode/ImagePoolProject/go.mod
@@ -0,0 +1,9 @@
+module ImagePoolProject
+
+go 1.13
+
+require (
+	github.com/astaxie/beego v1.12.0
+	github.com/go-redis/redis v6.15.6+incompatible
+	github.com/go-sql-driver/mysql v1.4.1
+)
diff --git a/imagePoolCode/ImagePoolProject/go.sum b/imagePoolCode/ImagePoolProject/go.sum
new file mode 100644
index 0000000..2375d40
--- /dev/null
+++ b/imagePoolCode/ImagePoolProject/go.sum
@@ -0,0 +1,39 @@
+github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
+github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM=
+github.com/astaxie/beego v1.12.0 h1:MRhVoeeye5N+Flul5PoVfD9CslfdoH+xqC/xvSQ5u2Y=
+github.com/astaxie/beego v1.12.0/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o=
+github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ=
+github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU=
+github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=
+github.com/casbin/casbin v1.7.0/go.mod h1:c67qKN6Oum3UF5Q1+BByfFxkwKvhwW57ITjqwtzR1KE=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
+github.com/couchbase/go-couchbase v0.0.0-20181122212707-3e9b6e1258bb/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U=
+github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c=
+github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
+github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY=
+github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
+github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-redis/redis v6.15.6+incompatible h1:H9evprGPLI8+ci7fxQx6WNZHJSb7be8FqJQRhdQZ5Sg=
+github.com/go-redis/redis v6.15.6+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
+github.com/siddontang/ledisdb v0.0.0-20181029004158-becf5f38d373/go.mod h1:mF1DpOSOUiJRMR+FDqaqu3EBqrybQtrDDszLUZ6oxPg=
+github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA=
+github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE=
+github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
+github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
+golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/imagePoolCode/ImagePoolProject/main.go b/imagePoolCode/ImagePoolProject/main.go
new file mode 100644
index 0000000..6fe6612
--- /dev/null
+++ b/imagePoolCode/ImagePoolProject/main.go
@@ -0,0 +1,24 @@
+package main
+
+import (
+	_ "fmt"
+	"github.com/astaxie/beego/orm"
+	_ "github.com/go-redis/redis"
+	_ "github.com/go-sql-driver/mysql"
+)
+
+type ModelInit struct {
+	Id   int
+}
+
+func init(){
+	orm.RegisterDataBase("default", "mysql", "root:116118@tcp(www.whatdoyoudo.club:3306)/dataset?charset=utf8", 30)
+	// create table
+	orm.RegisterModel(new(ModelInit))
+	orm.RunSyncdb("default", false, true)
+}
+
+func main()  {
+	GroupUrlGet("/m/014j1m")
+}
+

From 96a0d011baf51ab21dd714171de215a164643fd1 Mon Sep 17 00:00:00 2001
From: bert <1131482051@qq.com>
Date: Sat, 7 Dec 2019 20:17:15 +0800
Subject: [PATCH 3/3] 12.7 update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

更新下载器部分代码，解决图片下载损坏问题和次序问题
---
 imagePoolCode/ImagePoolProject/Downloader.go | 68 +++++++++++++++++---
 imagePoolCode/ImagePoolProject/main.go       | 64 +++++++++++++++---
 2 files changed, 114 insertions(+), 18 deletions(-)

diff --git a/imagePoolCode/ImagePoolProject/Downloader.go b/imagePoolCode/ImagePoolProject/Downloader.go
index 42fa542..0054444 100644
--- a/imagePoolCode/ImagePoolProject/Downloader.go
+++ b/imagePoolCode/ImagePoolProject/Downloader.go
@@ -11,15 +11,20 @@ import (
 	"strings"
 	"sync"
 )
+
+const (
+	ChanSize int = 5
+)
+
 //图片下载函数，保存图片到TempImage文件夹下
-func ImageDownload(imageID string ,url string )(status bool,err error){
+func ImageDownload(imageID string ,url string,label string)(status bool,err error){
 	resp,err := http.Get(url)
 	if err != nil {
 		fmt.Println(err)
 		return false,errors.New("can't connected")
 	}
 	if resp.StatusCode==200{
-		img,err:=os.OpenFile("./TempImage/"+imageID+".jpg",os.O_WRONLY|os.O_CREATE|os.O_EXCL,0666)
+		img,err:=os.OpenFile("./TempImage/"+label+"/"+imageID+".jpg",os.O_WRONLY|os.O_CREATE,0666)
 		defer img.Close()
 		if err != nil {
 			println(err)
@@ -30,7 +35,38 @@ func ImageDownload(imageID string ,url string )(status bool,err error){
 			if err == io.EOF {
 				break
 			}
-			n,err = img.Write(image[:n])
+			_,err = img.Write(image[:n])
+			if err != nil {
+				fmt.Println("img write:",err)
+			}
+		}
+		return true ,nil
+	} else{
+		err:=errors.New("StatusCode Error")
+		return false,err
+	}
+}
+func ImageDownload2(count chan int,url string,label string)(status bool,err error){
+	resp,err := http.Get(url)
+	if err != nil {
+		fmt.Println(err)
+		return false,errors.New("can't connected")
+	}
+	if resp.StatusCode==200{
+		a:=<-count
+		println(a,url)
+		img,err:=os.OpenFile("./TempImage/"+label+"/"+strconv.Itoa(a)+".jpg",os.O_WRONLY|os.O_CREATE,0777)
+		defer img.Close()
+		if err != nil {
+			println(err)
+		}
+		image:=make([]byte,256)
+		for {
+			n,err := resp.Body.Read(image)
+			if err == io.EOF {
+				break
+			}
+			_,err = img.Write(image[:n])
 			if err != nil {
 				fmt.Println("img write:",err)
 			}
@@ -51,18 +87,32 @@ func urlDownload(url string)[]string{
 	urls:=strings.Split(text,"\n")
 	return urls
 }
-func ImageNetDownload(urls []string){
+func ImageNetDownload(urls []string,label string)(int){
 	println(len(urls))
 	w:=sync.WaitGroup{}
-
+	lock:=sync.Mutex{}
+	sum :=9
+	count:=make(chan int,ChanSize)
+	for i:=0;i<ChanSize;i++{
+		count<-i
+	}
 	for i:=0;i<len(urls);i++{
 		w.Add(1)
-		println(i)
-		go func(i int,url string) {
+		go func(i int,url string,imgnum *int) {
 			url=strings.ReplaceAll(url,"\r","")
-			ImageDownload(strconv.Itoa(i),url)
+			flag,err:=ImageDownload2(count,url,label)
+			if err != nil {
+				fmt.Printf("%v\n",err)
+			}
+			if flag==true{
+				lock.Lock()
+				*imgnum++
+				count<-*imgnum
+				lock.Unlock()
+			}
 			w.Done()
-		}(i,urls[i])
+		}(i,urls[i],&sum)
 	}
 	w.Wait()
+	return sum
 }
\ No newline at end of file
diff --git a/imagePoolCode/ImagePoolProject/main.go b/imagePoolCode/ImagePoolProject/main.go
index 6fe6612..5a0a9ca 100644
--- a/imagePoolCode/ImagePoolProject/main.go
+++ b/imagePoolCode/ImagePoolProject/main.go
@@ -1,24 +1,70 @@
 package main
 
 import (
-	_ "fmt"
-	"github.com/astaxie/beego/orm"
+	_ "github.com/astaxie/beego/orm"
 	_ "github.com/go-redis/redis"
 	_ "github.com/go-sql-driver/mysql"
+	"os"
+	"strconv"
+	"strings"
 )
 
 type ModelInit struct {
 	Id   int
 }
 
-func init(){
-	orm.RegisterDataBase("default", "mysql", "root:116118@tcp(www.whatdoyoudo.club:3306)/dataset?charset=utf8", 30)
-	// create table
-	orm.RegisterModel(new(ModelInit))
-	orm.RunSyncdb("default", false, true)
+//func init(){
+//	orm.RegisterDataBase("default", "mysql", "root:116118@tcp(www.whatdoyoudo.club:3306)/dataset?charset=utf8", 30)
+//	// create table
+//	orm.RegisterModel(new(ModelInit))
+//	orm.RunSyncdb("default", false, true)
+//}
+
+func ImageNetDownloader(){
+	arg := os.Args
+	label := arg[1]
+	pwd,err:= os.Getwd()
+	if err != nil {
+
+	}
+	println(pwd)
+	err=os.MkdirAll(pwd+"TempImage",0777)
+	if err != nil {
+		println(err)
+	}
+	err=os.MkdirAll(pwd+"/TempImage/"+label,0777)
+	if err != nil {
+		println(err)
+	}
+	urls :=urlDownload("http://image-net.org/api/text/imagenet.synset.geturls?wnid="+label)
+	n:=ImageNetDownload(urls,label)
+	println("下载",n-10,"条"+label+"图像")
 }
 
-func main()  {
-	GroupUrlGet("/m/014j1m")
+func ImageNetDownloader2(){
+	arg := os.Args
+	label := arg[1]
+	pwd,err:= os.Getwd()
+	if err != nil {
+
+	}
+	println(pwd)
+	err=os.MkdirAll(pwd+"TempImage",0777)
+	if err != nil {
+		println(err)
+	}
+	err=os.MkdirAll(pwd+"/TempImage/"+label,0777)
+	if err != nil {
+		println(err)
+	}
+	urls :=urlDownload("http://image-net.org/api/text/imagenet.synset.geturls?wnid="+label)
+	for i,k:=range(urls){
+		println(i,k)
+		k=strings.ReplaceAll(k,"\r","")
+		ImageDownload(strconv.Itoa(i),k,label)
+	}
 }
 
+func main(){
+	ImageNetDownloader()
+}
\ No newline at end of file