Skip to content

Commit

Permalink
ImportCsvDialog: Allow other file encoding than UTF-8
Browse files Browse the repository at this point in the history 
Add a combobox and a text field for choosing a different encoding of the
input file than the default UTF-8.
  • Loading branch information
@MKleusberg
MKleusberg committed Aug 31, 2014
1 parent cf62b78 commit 03879d9
Show file tree
Hide file tree
Showing 6 changed files with 134 additions and 14 deletions.
27 changes: 23 additions & 4 deletions src/ImportCsvDialog.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
#include <QProgressDialog>
#include <QPushButton>
#include <QDateTime>
#include <QTextCodec>
#include <QCompleter>
#include <sqlite3.h>

ImportCsvDialog::ImportCsvDialog(const QString& filename, DBBrowserDB* db, QWidget* parent)
Expand All @@ -16,6 +18,13 @@ ImportCsvDialog::ImportCsvDialog(const QString& filename, DBBrowserDB* db, QWidg
{
ui->setupUi(this);

QStringList encodingList;
foreach(QString enc, QTextCodec::availableCodecs())
encodingList.push_back(enc);
encodingCompleter = new QCompleter(encodingList, this);
encodingCompleter->setCaseSensitivity(Qt::CaseInsensitive);
ui->editCustomEncoding->setCompleter(encodingCompleter);

checkInput();
updatePreview();
}
Expand All @@ -42,7 +51,7 @@ void ImportCsvDialog::accept()

// Parse all csv data
int numfields;
QStringList curList = pdb->decodeCSV(csvFilename, currentSeparatorChar(), currentQuoteChar(), -1, &numfields);
QStringList curList = pdb->decodeCSV(csvFilename, currentSeparatorChar(), currentQuoteChar(), currentEncoding(), -1, &numfields);

// Can not operate on an empty result
if(numfields == 0)
Expand Down Expand Up @@ -157,11 +166,12 @@ void ImportCsvDialog::updatePreview()
// Show/hide custom quote/separator input fields
ui->editCustomQuote->setVisible(ui->comboQuote->currentIndex() == ui->comboQuote->count()-1);
ui->editCustomSeparator->setVisible(ui->comboSeparator->currentIndex() == ui->comboSeparator->count()-1);
ui->editCustomEncoding->setVisible(ui->comboEncoding->currentIndex() == ui->comboEncoding->count()-1);

// Get preview data
int numfields;
int maxrecs = 20;
QStringList curList = pdb->decodeCSV(csvFilename, currentSeparatorChar(), currentQuoteChar(), maxrecs, &numfields);
QStringList curList = pdb->decodeCSV(csvFilename, currentSeparatorChar(), currentQuoteChar(), currentEncoding(), maxrecs, &numfields);

// Reset preview widget
ui->tablePreview->clear();
Expand Down Expand Up @@ -208,7 +218,7 @@ void ImportCsvDialog::checkInput()
ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(valid);
}

char ImportCsvDialog::currentQuoteChar()
char ImportCsvDialog::currentQuoteChar() const
{
// The last item in the combobox is the 'Other' item; if it is selected return the text of the line edit field instead
if(ui->comboQuote->currentIndex() == ui->comboQuote->count()-1)
Expand All @@ -220,11 +230,20 @@ char ImportCsvDialog::currentQuoteChar()
return 0;
}

char ImportCsvDialog::currentSeparatorChar()
char ImportCsvDialog::currentSeparatorChar() const
{
// The last item in the combobox is the 'Other' item; if it is selected return the text of the line edit field instead
if(ui->comboSeparator->currentIndex() == ui->comboSeparator->count()-1)
return ui->editCustomSeparator->text().length() ? ui->editCustomSeparator->text().at(0).toLatin1() : 0;

return ui->comboSeparator->currentText() == tr("Tab") ? '\t' : ui->comboSeparator->currentText().at(0).toLatin1();
}

QString ImportCsvDialog::currentEncoding() const
{
// The last item in the combobox is the 'Other' item; if it is selected return the text of the line edit field instead
if(ui->comboEncoding->currentIndex() == ui->comboEncoding->count()-1)
return ui->editCustomEncoding->text().length() ? ui->editCustomEncoding->text() : "UTF-8";
else
return ui->comboEncoding->currentText();
}
7 changes: 5 additions & 2 deletions src/ImportCsvDialog.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <QDialog>

class DBBrowserDB;
class QCompleter;

namespace Ui {
class ImportCsvDialog;
Expand All @@ -26,9 +27,11 @@ private slots:
Ui::ImportCsvDialog* ui;
QString csvFilename;
DBBrowserDB* pdb;
QCompleter* encodingCompleter;

char currentQuoteChar();
char currentSeparatorChar();
char currentQuoteChar() const;
char currentSeparatorChar() const;
QString currentEncoding() const;
};

#endif
97 changes: 92 additions & 5 deletions src/ImportCsvDialog.ui
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,60 @@
</item>
</layout>
</item>
<item row="4" column="0">
<widget class="QLabel" name="labelEncoding">
<property name="text">
<string>&amp;Encoding</string>
</property>
<property name="buddy">
<cstring>comboEncoding</cstring>
</property>
</widget>
</item>
<item row="4" column="1">
<layout class="QHBoxLayout" name="horizontalLayout_3">
<item>
<widget class="QComboBox" name="comboEncoding">
<item>
<property name="text">
<string>UTF-8</string>
</property>
</item>
<item>
<property name="text">
<string>UTF-16</string>
</property>
</item>
<item>
<property name="text">
<string>ISO-8859-1</string>
</property>
</item>
<item>
<property name="text">
<string>Other</string>
</property>
</item>
</widget>
</item>
<item>
<widget class="QLineEdit" name="editCustomEncoding"/>
</item>
<item>
<spacer name="horizontalSpacer_3">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
</layout>
</item>
</layout>
</item>
<item>
Expand All @@ -194,8 +248,9 @@
<tabstop>editCustomSeparator</tabstop>
<tabstop>comboQuote</tabstop>
<tabstop>editCustomQuote</tabstop>
<tabstop>comboEncoding</tabstop>
<tabstop>editCustomEncoding</tabstop>
<tabstop>tablePreview</tabstop>
<tabstop>buttonBox</tabstop>
</tabstops>
<resources/>
<connections>
Expand All @@ -206,8 +261,8 @@
<slot>accept()</slot>
<hints>
<hint type="sourcelabel">
<x>252</x>
<y>485</y>
<x>261</x>
<y>480</y>
</hint>
<hint type="destinationlabel">
<x>157</x>
Expand All @@ -222,8 +277,8 @@
<slot>reject()</slot>
<hints>
<hint type="sourcelabel">
<x>320</x>
<y>485</y>
<x>329</x>
<y>480</y>
</hint>
<hint type="destinationlabel">
<x>286</x>
Expand Down Expand Up @@ -327,6 +382,38 @@
</hint>
</hints>
</connection>
<connection>
<sender>comboEncoding</sender>
<signal>currentIndexChanged(int)</signal>
<receiver>ImportCsvDialog</receiver>
<slot>updatePreview()</slot>
<hints>
<hint type="sourcelabel">
<x>198</x>
<y>131</y>
</hint>
<hint type="destinationlabel">
<x>572</x>
<y>121</y>
</hint>
</hints>
</connection>
<connection>
<sender>editCustomEncoding</sender>
<signal>textChanged(QString)</signal>
<receiver>ImportCsvDialog</receiver>
<slot>updatePreview()</slot>
<hints>
<hint type="sourcelabel">
<x>378</x>
<y>132</y>
</hint>
<hint type="destinationlabel">
<x>540</x>
<y>133</y>
</hint>
</hints>
</connection>
</connections>
<slots>
<slot>updatePreview()</slot>
Expand Down
3 changes: 2 additions & 1 deletion src/sqlitedb.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -873,7 +873,7 @@ void DBBrowserDB::updateSchema( )
}
}

QStringList DBBrowserDB::decodeCSV(const QString & csvfilename, char sep, char quote, int maxrecords, int * numfields)
QStringList DBBrowserDB::decodeCSV(const QString & csvfilename, char sep, char quote, const QString& encoding, int maxrecords, int * numfields)
{
QFile file(csvfilename);
QStringList result;
Expand All @@ -886,6 +886,7 @@ QStringList DBBrowserDB::decodeCSV(const QString & csvfilename, char sep, char q

//Other than QFile, the QTextStream-class properly detects 2-Byte QChars and converts them accordingly (UTF-8)
QTextStream inStream(&file);
inStream.setCodec(encoding.toUtf8());

QProgressDialog progress(QObject::tr("Decoding CSV file..."), QObject::tr("Cancel"), 0, file.size());
progress.setWindowModality(Qt::ApplicationModal);
Expand Down
2 changes: 1 addition & 1 deletion src/sqlitedb.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ class DBBrowserDB : public QObject

sqlite3 * _db;

QStringList decodeCSV(const QString & csvfilename, char sep, char quote, int maxrecords, int * numfields);
QStringList decodeCSV(const QString & csvfilename, char sep, char quote, const QString& encoding, int maxrecords, int * numfields);

objectMap objMap;

Expand Down
12 changes: 11 additions & 1 deletion src/tests/TestImport.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ void TestImport::csvImport()
QFETCH(QString, csv);
QFETCH(char, separator);
QFETCH(char, quote);
QFETCH(QString, encoding);
QFETCH(int, numfields);
QFETCH(QStringList, result);

Expand All @@ -28,7 +29,7 @@ void TestImport::csvImport()
// Call decodeCSV function
DBBrowserDB db;
int numfields_read;
QStringList retval = db.decodeCSV(file.fileName(), separator, quote, -1, &numfields_read);
QStringList retval = db.decodeCSV(file.fileName(), separator, quote, encoding, -1, &numfields_read);

// Check return values
QCOMPARE(retval, result);
Expand All @@ -40,6 +41,7 @@ void TestImport::csvImport_data()
QTest::addColumn<QString>("csv");
QTest::addColumn<char>("separator");
QTest::addColumn<char>("quote");
QTest::addColumn<QString>("encoding");
QTest::addColumn<int>("numfields");
QTest::addColumn<QStringList>("result");

Expand All @@ -48,26 +50,31 @@ void TestImport::csvImport_data()
QTest::newRow("commas_noquotes") << "a,b,c\nd,e,f\ng,h,i\n"
<< ','
<< (char)0
<< "UTF-8"
<< 3
<< result;
QTest::newRow("semicolons_noquotes") << "a;b;c\nd;e;f\ng;h;i\n"
<< ';'
<< (char)0
<< "UTF-8"
<< 3
<< result;
QTest::newRow("commas_doublequotes") << "\"a\",\"b\",\"c\"\n\"d\",\"e\",\"f\"\n\"g\",\"h\",\"i\"\n"
<< ','
<< '"'
<< "UTF-8"
<< 3
<< result;
QTest::newRow("noquotes_butquotesset") << "a,b,c\nd,e,f\ng,h,i\n"
<< ','
<< '"'
<< "UTF-8"
<< 3
<< result;
QTest::newRow("windowslinebreaks") << "a,b,c\r\nd,e,f\r\ng,h,i\r\n"
<< ','
<< (char)0
<< "UTF-8"
<< 3
<< result;

Expand All @@ -76,6 +83,7 @@ void TestImport::csvImport_data()
QTest::newRow("oneline") << "a,b,c"
<< ','
<< (char)0
<< "UTF-8"
<< 3
<< result;

Expand All @@ -84,6 +92,7 @@ void TestImport::csvImport_data()
QTest::newRow("manyquotes") << "\"a,a\"\"\",\"b\",\"c\"\n\"d\",\"e\",\"\"\"\"\"f,f\"\n"
<< ','
<< '"'
<< "UTF-8"
<< 3
<< result;

Expand All @@ -93,6 +102,7 @@ void TestImport::csvImport_data()
QTest::newRow("utf8chars") << csv
<< ','
<< (char)0
<< "UTF-8"
<< 3
<< result;
}

0 comments on commit 03879d9

Please sign in to comment.