在mysql中去重，其实是一个很简单的事情，来看下面的例子：

mysql> DROP TABLE test;

Query OK, 0 rows affected (0.01 sec)

mysql> CREATE TABLE test
(
id INT,
NAME VARCHAR(10)
);
Query OK, 0 rows affected (0.01 sec)

mysql> INSERT INTO test VALUES(1,'a1'),(2,'a2'),(3,'a3'),(4,'a4'),(1,'a1');
Query OK, 5 rows affected (0.00 sec)
Records: 5 Duplicates: 0 Warnings: 0

mysql> SELECT * FROM test;
+------+------+
| id | NAME |
+------+------+
| 1 | a1 |
| 2 | a2 |
| 3 | a3 |
| 4 | a4 |
| 1 | a1 |
+------+------+
5 rows in set (0.00 sec)

mysql> ALTER IGNORE TABLE test ADD UNIQUE INDEX idx_id_name (id,NAME);
Query OK, 5 rows affected, 1 warning (0.02 sec)
Records: 5 Duplicates: 1 Warnings: 1

mysql> SHOW WARNINGS;
+---------+------+-----------------------------------------------------------------+
| Level | Code | Message |
+---------+------+-----------------------------------------------------------------+
| Warning | 1681 | 'IGNORE' is deprecated and will be removed in a future release. |
+---------+------+-----------------------------------------------------------------+
1 row in set (0.00 sec)

mysql> SELECT * FROM test;
+------+------+
| id | NAME |
+------+------+
| 1 | a1 |
| 2 | a2 |
| 3 | a3 |
| 4 | a4 |
+------+------+
4 rows in set (0.00 sec)