diff --git a/docs-core/pom.xml b/docs-core/pom.xml index dcde9a75..0bf5cf36 100644 --- a/docs-core/pom.xml +++ b/docs-core/pom.xml @@ -117,6 +117,22 @@ imgscalr-lib + + + jna + jna + + + + jai + imageio + + + + tess4j + tess4j + + junit diff --git a/docs-core/src/main/java/com/sismics/docs/core/constant/Constants.java b/docs-core/src/main/java/com/sismics/docs/core/constant/Constants.java index bc7fe87f..30312918 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/constant/Constants.java +++ b/docs-core/src/main/java/com/sismics/docs/core/constant/Constants.java @@ -1,5 +1,9 @@ package com.sismics.docs.core.constant; +import java.util.List; + +import com.google.common.collect.Lists; + /** * Application constants. * @@ -40,4 +44,9 @@ public class Constants { * Default generic user role. */ public static final String DEFAULT_USER_ROLE = "user"; + + /** + * Supported document languages. + */ + public static final List SUPPORTED_LANGUAGES = Lists.newArrayList("eng", "fra"); } diff --git a/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/DocumentDao.java b/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/DocumentDao.java index 4397b7f6..90249978 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/DocumentDao.java +++ b/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/DocumentDao.java @@ -123,7 +123,7 @@ public class DocumentDao { Map parameterMap = new HashMap(); List criteriaList = new ArrayList(); - StringBuilder sb = new StringBuilder("select d.DOC_ID_C c0, d.DOC_TITLE_C c1, d.DOC_DESCRIPTION_C c2, d.DOC_CREATEDATE_D c3, s.SHA_ID_C is not null c4 "); + StringBuilder sb = new StringBuilder("select d.DOC_ID_C c0, d.DOC_TITLE_C c1, d.DOC_DESCRIPTION_C c2, d.DOC_CREATEDATE_D c3, d.DOC_LANGUAGE_C c4, s.SHA_ID_C is not null c5 "); sb.append(" from T_DOCUMENT d "); sb.append(" left join T_SHARE s on s.SHA_IDDOCUMENT_C = d.DOC_ID_C and s.SHA_DELETEDATE_D is null "); @@ -156,6 +156,10 @@ public class DocumentDao { if (criteria.getShared() != null && criteria.getShared()) { criteriaList.add("s.SHA_ID_C is not null"); } + if (criteria.getLanguage() != null) { + criteriaList.add("d.DOC_LANGUAGE_C = :language"); + parameterMap.put("language", criteria.getLanguage()); + } criteriaList.add("d.DOC_DELETEDATE_D is null"); @@ -177,6 +181,7 @@ public class DocumentDao { documentDto.setTitle((String) o[i++]); documentDto.setDescription((String) o[i++]); documentDto.setCreateTimestamp(((Timestamp) o[i++]).getTime()); + documentDto.setLanguage((String) o[i++]); documentDto.setShared((Boolean) o[i++]); documentDtoList.add(documentDto); } diff --git a/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/FileDao.java b/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/FileDao.java index 74b3a57a..0eee780f 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/FileDao.java +++ b/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/FileDao.java @@ -1,14 +1,15 @@ package com.sismics.docs.core.dao.jpa; -import com.sismics.docs.core.model.jpa.File; -import com.sismics.util.context.ThreadLocalContext; +import java.util.Date; +import java.util.List; +import java.util.UUID; import javax.persistence.EntityManager; import javax.persistence.NoResultException; import javax.persistence.Query; -import java.util.Date; -import java.util.List; -import java.util.UUID; + +import com.sismics.docs.core.model.jpa.File; +import com.sismics.util.context.ThreadLocalContext; /** * File DAO. @@ -66,6 +67,26 @@ public class FileDao { fileDb.setDeleteDate(dateNow); } + /** + * Updates the content of a file. + * + * @param file File to update + * @return Updated file + */ + public File updateContent(File file) { + EntityManager em = ThreadLocalContext.get().getEntityManager(); + + // Get the file + Query q = em.createQuery("select f from File f where f.id = :id and f.deleteDate is null"); + q.setParameter("id", file.getId()); + File fileFromDb = (File) q.getSingleResult(); + + // Update the user + fileFromDb.setContent(file.getContent()); + + return file; + } + /** * Gets a file by its ID. * diff --git a/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/criteria/DocumentCriteria.java b/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/criteria/DocumentCriteria.java index 0419afc8..2b61b339 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/criteria/DocumentCriteria.java +++ b/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/criteria/DocumentCriteria.java @@ -40,6 +40,11 @@ public class DocumentCriteria { */ private Boolean shared; + /** + * Language. + */ + private String language; + /** * Getter of userId. * @@ -147,4 +152,22 @@ public class DocumentCriteria { public void setShared(Boolean shared) { this.shared = shared; } + + /** + * Getter of language. + * + * @return the language + */ + public String getLanguage() { + return language; + } + + /** + * Setter of language. + * + * @param language language + */ + public void setLanguage(String language) { + this.language = language; + } } diff --git a/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/dto/DocumentDto.java b/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/dto/DocumentDto.java index e083965e..289ff400 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/dto/DocumentDto.java +++ b/docs-core/src/main/java/com/sismics/docs/core/dao/jpa/dto/DocumentDto.java @@ -24,6 +24,11 @@ public class DocumentDto { */ private String description; + /** + * Language. + */ + private String language; + /** * Creation date. */ @@ -123,4 +128,22 @@ public class DocumentDto { public void setShared(Boolean shared) { this.shared = shared; } + + /** + * Getter of language. + * + * @return the language + */ + public String getLanguage() { + return language; + } + + /** + * Setter of language. + * + * @param language language + */ + public void setLanguage(String language) { + this.language = language; + } } diff --git a/docs-core/src/main/java/com/sismics/docs/core/dao/lucene/LuceneDao.java b/docs-core/src/main/java/com/sismics/docs/core/dao/lucene/LuceneDao.java new file mode 100644 index 00000000..6b5a72a9 --- /dev/null +++ b/docs-core/src/main/java/com/sismics/docs/core/dao/lucene/LuceneDao.java @@ -0,0 +1,151 @@ +package com.sismics.docs.core.dao.lucene; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.queries.TermsFilter; +import org.apache.lucene.queryparser.flexible.standard.QueryParserUtil; +import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.util.Version; + +import com.sismics.docs.core.model.context.AppContext; +import com.sismics.docs.core.model.jpa.File; +import com.sismics.docs.core.util.LuceneUtil; +import com.sismics.docs.core.util.LuceneUtil.LuceneRunnable; + +/** + * Lucene DAO. + * + * @author bgamard + */ +public class LuceneDao { + + /** + * Destroy and rebuild index. + * + * @param fileList + */ + public void rebuildIndex(final List fileList) { + LuceneUtil.handle(new LuceneRunnable() { + @Override + public void run(IndexWriter indexWriter) throws Exception { + // Empty index + indexWriter.deleteAll(); + + // Add all files + for (File file : fileList) { + org.apache.lucene.document.Document document = getDocumentFromFile(file); + indexWriter.addDocument(document); + } + } + }); + } + + + /** + * Add files to the index. + * + * @param fileList + */ + public void create(final List fileList) { + LuceneUtil.handle(new LuceneRunnable() { + @Override + public void run(IndexWriter indexWriter) throws Exception { + // Add all files + for (File file : fileList) { + org.apache.lucene.document.Document document = getDocumentFromFile(file); + indexWriter.addDocument(document); + } + } + }); + } + + /** + * Update index. + * + * @param fileList File list + */ + public void update(final List fileList) { + LuceneUtil.handle(new LuceneRunnable() { + @Override + public void run(IndexWriter indexWriter) throws Exception { + // Update all files + for (File file : fileList) { + org.apache.lucene.document.Document document = getDocumentFromFile(file); + indexWriter.updateDocument(new Term("id", file.getId()), document); + } + } + }); + } + + /** + * Search files. + * + * @param paginatedList + * @param feedList + * @param searchQuery + * @return List of file IDs + * @throws Exception + */ + public List search(String userId, String searchQuery, int limit) throws Exception { + // Escape query and add quotes so QueryParser generate a PhraseQuery + searchQuery = "\"" + QueryParserUtil.escape(searchQuery) + "\""; + + // Build search query + StandardQueryParser qpHelper = new StandardQueryParser(new DocsStandardAnalyzer(Version.LUCENE_42)); + qpHelper.setPhraseSlop(100000); // PhraseQuery add terms + Query contentQuery = qpHelper.parse(searchQuery, "content"); + + // Search on file content + BooleanQuery query = new BooleanQuery(); + query.add(contentQuery, Occur.SHOULD); + + // Filter on provided user ID + List terms = new ArrayList(); + terms.add(new Term("user_id", userId)); + TermsFilter feedsFilter = new TermsFilter(terms); + + // Search + IndexReader reader = DirectoryReader.open(AppContext.getInstance().getLuceneDirectory()); + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(query, feedsFilter, limit); + ScoreDoc[] docs = topDocs.scoreDocs; + + // Extract file IDs + List fileIdList = new ArrayList(); + for (int i = 0; i < docs.length; i++) { + String id = searcher.doc(docs[i].doc).get("id"); + fileIdList.add(id); + } + + return fileIdList; + } + + /** + * Build Lucene document from file. + * + * @param file File + * @return Document + */ + private org.apache.lucene.document.Document getDocumentFromFile(File file) { + // Building document + org.apache.lucene.document.Document document = new org.apache.lucene.document.Document(); + document.add(new StringField("id", file.getId(), Field.Store.YES)); + document.add(new TextField("content", file.getContent(), Field.Store.NO)); + + return document; + } +} diff --git a/docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java b/docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java new file mode 100644 index 00000000..cdbc2976 --- /dev/null +++ b/docs-core/src/main/java/com/sismics/docs/core/event/FileCreatedAsyncEvent.java @@ -0,0 +1,66 @@ +package com.sismics.docs.core.event; + +import com.google.common.base.Objects; +import com.sismics.docs.core.model.jpa.Document; +import com.sismics.docs.core.model.jpa.File; + +/** + * New file created event. + * + * @author bgamard + */ +public class FileCreatedAsyncEvent { + /** + * Created file. + */ + private File file; + + /** + * Document linked to the file. + */ + private Document document; + + /** + * Getter of file. + * + * @return the file + */ + public File getFile() { + return file; + } + + /** + * Setter of file. + * + * @param file file + */ + public void setFile(File file) { + this.file = file; + } + + /** + * Getter of document. + * + * @return the document + */ + public Document getDocument() { + return document; + } + + /** + * Setter of document. + * + * @param document document + */ + public void setDocument(Document document) { + this.document = document; + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("file", file) + .add("document", document) + .toString(); + } +} \ No newline at end of file diff --git a/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java b/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java new file mode 100644 index 00000000..c25e7d37 --- /dev/null +++ b/docs-core/src/main/java/com/sismics/docs/core/listener/async/FileCreatedAsyncListener.java @@ -0,0 +1,43 @@ +package com.sismics.docs.core.listener.async; + +import java.text.MessageFormat; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.eventbus.Subscribe; +import com.sismics.docs.core.event.FileCreatedAsyncEvent; +import com.sismics.docs.core.util.FileUtil; +import com.sismics.util.ImageUtil; + +/** + * Listener on new file. + * + * @author bgamard + */ +public class FileCreatedAsyncListener { + /** + * Logger. + */ + private static final Logger log = LoggerFactory.getLogger(FileCreatedAsyncListener.class); + + /** + * Process new file. + * + * @param fileCreatedAsyncEvent New file created event + * @throws Exception + */ + @Subscribe + public void onArticleCreated(final FileCreatedAsyncEvent fileCreatedAsyncEvent) throws Exception { + if (log.isInfoEnabled()) { + log.info("File created event: " + fileCreatedAsyncEvent.toString()); + } + + // OCR the file if it is an image + if (ImageUtil.isImage(fileCreatedAsyncEvent.getFile().getMimeType())) { + long startTime = System.currentTimeMillis(); + FileUtil.ocrFile(fileCreatedAsyncEvent.getDocument(), fileCreatedAsyncEvent.getFile()); + log.info(MessageFormat.format("File OCR-ized in {0}ms", System.currentTimeMillis() - startTime)); + } + } +} diff --git a/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java b/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java index 9b5b2bab..e0bae3e1 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java +++ b/docs-core/src/main/java/com/sismics/docs/core/model/context/AppContext.java @@ -1,15 +1,5 @@ package com.sismics.docs.core.model.context; -import com.google.common.eventbus.AsyncEventBus; -import com.google.common.eventbus.EventBus; -import com.sismics.docs.core.constant.ConfigType; -import com.sismics.docs.core.dao.jpa.ConfigDao; -import com.sismics.docs.core.listener.sync.DeadEventListener; -import com.sismics.docs.core.model.jpa.Config; -import com.sismics.docs.core.service.IndexingService; -import com.sismics.util.EnvironmentUtil; -import org.apache.lucene.store.Directory; - import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; @@ -17,6 +7,18 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.lucene.store.Directory; + +import com.google.common.eventbus.AsyncEventBus; +import com.google.common.eventbus.EventBus; +import com.sismics.docs.core.constant.ConfigType; +import com.sismics.docs.core.dao.jpa.ConfigDao; +import com.sismics.docs.core.listener.async.FileCreatedAsyncListener; +import com.sismics.docs.core.listener.sync.DeadEventListener; +import com.sismics.docs.core.model.jpa.Config; +import com.sismics.docs.core.service.IndexingService; +import com.sismics.util.EnvironmentUtil; + /** * Global application context. * @@ -77,6 +79,7 @@ public class AppContext { asyncExecutorList = new ArrayList(); asyncEventBus = newAsyncEventBus(); + asyncEventBus.register(new FileCreatedAsyncListener()); } /** diff --git a/docs-core/src/main/java/com/sismics/docs/core/model/jpa/Document.java b/docs-core/src/main/java/com/sismics/docs/core/model/jpa/Document.java index ccebce2c..4622f4fe 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/model/jpa/Document.java +++ b/docs-core/src/main/java/com/sismics/docs/core/model/jpa/Document.java @@ -29,6 +29,12 @@ public class Document { @Column(name = "DOC_IDUSER_C", nullable = false, length = 36) private String userId; + /** + * Language (ISO 639-9). + */ + @Column(name = "DOC_LANGUAGE_C", nullable = false, length = 3) + private String language; + /** * Title. */ @@ -71,6 +77,24 @@ public class Document { this.id = id; } + /** + * Getter of language. + * + * @return the language + */ + public String getLanguage() { + return language; + } + + /** + * Setter of language. + * + * @param language language + */ + public void setLanguage(String language) { + this.language = language; + } + /** * Getter of userId. * diff --git a/docs-core/src/main/java/com/sismics/docs/core/model/jpa/File.java b/docs-core/src/main/java/com/sismics/docs/core/model/jpa/File.java index a483e0c3..d1d46f4e 100644 --- a/docs-core/src/main/java/com/sismics/docs/core/model/jpa/File.java +++ b/docs-core/src/main/java/com/sismics/docs/core/model/jpa/File.java @@ -5,6 +5,7 @@ import com.google.common.base.Objects; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; +import javax.persistence.Lob; import javax.persistence.Table; import java.util.Date; @@ -30,11 +31,18 @@ public class File { private String documentId; /** - * Document ID. + * MIME type. */ @Column(name = "FIL_MIMETYPE_C", length = 100) private String mimeType; + /** + * OCR-ized content. + */ + @Lob + @Column(name = "FIL_CONTENT_C") + private String content; + /** * Creation date. */ @@ -143,6 +151,24 @@ public class File { this.deleteDate = deleteDate; } + /** + * Getter of content. + * + * @return the content + */ + public String getContent() { + return content; + } + + /** + * Setter of content. + * + * @param content content + */ + public void setContent(String content) { + this.content = content; + } + /** * Getter of order. * diff --git a/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java new file mode 100644 index 00000000..c896950c --- /dev/null +++ b/docs-core/src/main/java/com/sismics/docs/core/util/FileUtil.java @@ -0,0 +1,75 @@ +package com.sismics.docs.core.util; + +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.nio.file.Paths; + +import javax.imageio.ImageIO; + +import net.sourceforge.tess4j.Tesseract; + +import org.imgscalr.Scalr; +import org.imgscalr.Scalr.Method; +import org.imgscalr.Scalr.Mode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.sismics.docs.core.dao.jpa.FileDao; +import com.sismics.docs.core.model.jpa.Document; +import com.sismics.docs.core.model.jpa.File; + +/** + * File entity utilities. + * + * @author bgamard + */ +public class FileUtil { + /** + * Logger. + */ + private static final Logger log = LoggerFactory.getLogger(FileUtil.class); + + /** + * OCR a file. + * + * @param document Document linked to the file + * @param file File to OCR + */ + public static void ocrFile(Document document, final File file) { + Tesseract instance = Tesseract.getInstance(); + java.io.File storedfile = Paths.get(DirectoryUtil.getStorageDirectory().getPath(), file.getId()).toFile(); + String content = null; + BufferedImage image = null; + try { + image = ImageIO.read(storedfile); + } catch (IOException e) { + log.error("Error reading the image " + storedfile, e); + } + + // Upscale the image if it is too small + if (image.getWidth() < 2500 || image.getHeight() < 2500) { + BufferedImage resizedImage = Scalr.resize(image, Method.AUTOMATIC, Mode.AUTOMATIC, 3500); + image.flush(); + image = resizedImage; + } + + // OCR the file + try { + instance.setLanguage(document.getLanguage()); + content = instance.doOCR(image); + } catch (Exception e) { + log.error("Error while OCR-izing the file " + storedfile, e); + } + + file.setContent(content); + + // Store the OCR-ization result in the database + TransactionUtil.handle(new Runnable() { + @Override + public void run() { + FileDao fileDao = new FileDao(); + fileDao.updateContent(file); + } + }); + } +} diff --git a/docs-core/src/main/resources/config.properties b/docs-core/src/main/resources/config.properties index 03d382c6..a8182ebd 100644 --- a/docs-core/src/main/resources/config.properties +++ b/docs-core/src/main/resources/config.properties @@ -1 +1 @@ -db.version=4 \ No newline at end of file +db.version=5 \ No newline at end of file diff --git a/docs-core/src/main/resources/db/update/dbupdate-005-0.sql b/docs-core/src/main/resources/db/update/dbupdate-005-0.sql new file mode 100644 index 00000000..a6465ec1 --- /dev/null +++ b/docs-core/src/main/resources/db/update/dbupdate-005-0.sql @@ -0,0 +1,3 @@ +alter table T_FILE add column FIL_CONTENT_C LONGVARCHAR; +alter table T_DOCUMENT add column DOC_LANGUAGE_C varchar(3) default 'fra' not null; +update T_CONFIG set CFG_VALUE_C='5' where CFG_ID_C='DB_VERSION'; diff --git a/docs-parent/TODO b/docs-parent/TODO index e69de29b..f6d835e9 100644 --- a/docs-parent/TODO +++ b/docs-parent/TODO @@ -0,0 +1,7 @@ +- Add language on document (client) +- Index title and description (server) +- Use Lucene for title and description searching (server) +- Index OCR-ized content (server) +- Search in OCR-ized files (server) +- Batch to OCR all documents (server) +- Batch to rebuild Lucene index (server) \ No newline at end of file diff --git a/docs-parent/lib/jai_imageio.jar b/docs-parent/lib/jai_imageio.jar new file mode 100644 index 00000000..571aa199 Binary files /dev/null and b/docs-parent/lib/jai_imageio.jar differ diff --git a/docs-parent/lib/jna.jar b/docs-parent/lib/jna.jar new file mode 100644 index 00000000..0827fcfa Binary files /dev/null and b/docs-parent/lib/jna.jar differ diff --git a/docs-parent/lib/tess4j.jar b/docs-parent/lib/tess4j.jar new file mode 100644 index 00000000..2ab9b13b Binary files /dev/null and b/docs-parent/lib/tess4j.jar differ diff --git a/docs-parent/pom.xml b/docs-parent/pom.xml index e61280c4..a033fb5c 100644 --- a/docs-parent/pom.xml +++ b/docs-parent/pom.xml @@ -159,6 +159,7 @@ osxappbundle-maven-plugin ${org.codehaus.mojo.osxappbundle-maven-plugin.version} + @@ -434,6 +435,25 @@ imgscalr-lib ${org.imgscalr.imgscalr-lib.version} + + + + jna + jna + 1.0 + + + + jai + imageio + 1.0 + + + + tess4j + tess4j + 1.0 + @@ -450,4 +470,74 @@ + + + + init + + + + + org.apache.maven.plugins + maven-install-plugin + 2.3.1 + + + + install-jna + validate + + ${project.basedir}/lib/jna.jar + default + jna + jna + 1.0 + jar + true + + + install-file + + + + + install-jai-imageio + validate + + ${project.basedir}/lib/jai_imageio.jar + default + jai + imageio + 1.0 + jar + true + + + install-file + + + + + install-tess4j + validate + + ${project.basedir}/lib/tess4j.jar + default + tess4j + tess4j + 1.0 + jar + true + + + install-file + + + + + + + + + diff --git a/docs-web/src/dev/resources/config.properties b/docs-web/src/dev/resources/config.properties index 299540e8..680c36e5 100644 --- a/docs-web/src/dev/resources/config.properties +++ b/docs-web/src/dev/resources/config.properties @@ -1,3 +1,3 @@ api.current_version=${project.version} api.min_version=1.0 -db.version=4 \ No newline at end of file +db.version=5 \ No newline at end of file diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java index d9a4c37d..38509e0c 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/DocumentResource.java @@ -31,6 +31,7 @@ import org.joda.time.format.DateTimeFormatterBuilder; import org.joda.time.format.DateTimeParser; import com.google.common.base.Strings; +import com.sismics.docs.core.constant.Constants; import com.sismics.docs.core.dao.jpa.DocumentDao; import com.sismics.docs.core.dao.jpa.ShareDao; import com.sismics.docs.core.dao.jpa.TagDao; @@ -155,6 +156,7 @@ public class DocumentResource extends BaseResource { document.put("description", documentDto.getDescription()); document.put("create_date", documentDto.getCreateTimestamp()); document.put("shared", documentDto.getShared()); + document.put("language", documentDto.getLanguage()); // Get tags List tagDtoList = tagDao.getByDocumentId(documentDto.getId()); @@ -178,7 +180,7 @@ public class DocumentResource extends BaseResource { /** * Parse a query according to the specified syntax, eg.: - * tag:assurance tag:other before:2012 after:2011-09 shared:yes thing + * tag:assurance tag:other before:2012 after:2011-09 shared:yes lang:fra thing * * @param search Search query * @return DocumentCriteria @@ -233,6 +235,11 @@ public class DocumentResource extends BaseResource { if (params[1].equals("yes")) { documentCriteria.setShared(true); } + } else if (params[0].equals("lang")) { + // New shared state criteria + if (Constants.SUPPORTED_LANGUAGES.contains(params[1])) { + documentCriteria.setLanguage(params[1]); + } } else { query.append(criteria); } @@ -256,6 +263,7 @@ public class DocumentResource extends BaseResource { @FormParam("title") String title, @FormParam("description") String description, @FormParam("tags") List tagList, + @FormParam("language") String language, @FormParam("create_date") String createDateStr) throws JSONException { if (!authenticate()) { throw new ForbiddenClientException(); @@ -263,8 +271,12 @@ public class DocumentResource extends BaseResource { // Validate input data title = ValidationUtil.validateLength(title, "title", 1, 100, false); + language = ValidationUtil.validateLength(language, "language", 3, 3, false); description = ValidationUtil.validateLength(description, "description", 0, 4000, true); Date createDate = ValidationUtil.validateDate(createDateStr, "create_date", true); + if (!Constants.SUPPORTED_LANGUAGES.contains(language)) { + throw new ClientException("ValidationError", MessageFormat.format("{0} is not a supported language", language)); + } // Create the document DocumentDao documentDao = new DocumentDao(); @@ -272,6 +284,7 @@ public class DocumentResource extends BaseResource { document.setUserId(principal.getId()); document.setTitle(title); document.setDescription(description); + document.setLanguage(language); if (createDate == null) { document.setCreateDate(new Date()); } else { @@ -303,6 +316,7 @@ public class DocumentResource extends BaseResource { @FormParam("title") String title, @FormParam("description") String description, @FormParam("tags") List tagList, + @FormParam("language") String language, @FormParam("create_date") String createDateStr) throws JSONException { if (!authenticate()) { throw new ForbiddenClientException(); @@ -310,8 +324,12 @@ public class DocumentResource extends BaseResource { // Validate input data title = ValidationUtil.validateLength(title, "title", 1, 100, true); + language = ValidationUtil.validateLength(language, "language", 3, 3, true); description = ValidationUtil.validateLength(description, "description", 0, 4000, true); Date createDate = ValidationUtil.validateDate(createDateStr, "create_date", true); + if (language != null && !Constants.SUPPORTED_LANGUAGES.contains(language)) { + throw new ClientException("ValidationError", MessageFormat.format("{0} is not a supported language", language)); + } // Get the document DocumentDao documentDao = new DocumentDao(); @@ -332,6 +350,9 @@ public class DocumentResource extends BaseResource { if (createDate != null) { document.setCreateDate(createDate); } + if (language != null) { + document.setLanguage(language); + } // Update tags updateTagList(id, tagList); diff --git a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java index 06f82334..2a8e4d18 100644 --- a/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java +++ b/docs-web/src/main/java/com/sismics/docs/rest/resource/FileResource.java @@ -29,6 +29,8 @@ import org.codehaus.jettison.json.JSONObject; import com.sismics.docs.core.dao.jpa.DocumentDao; import com.sismics.docs.core.dao.jpa.FileDao; import com.sismics.docs.core.dao.jpa.ShareDao; +import com.sismics.docs.core.event.FileCreatedAsyncEvent; +import com.sismics.docs.core.model.context.AppContext; import com.sismics.docs.core.model.jpa.Document; import com.sismics.docs.core.model.jpa.File; import com.sismics.docs.core.util.DirectoryUtil; @@ -110,6 +112,12 @@ public class FileResource extends BaseResource { // Save the file FileUtil.save(is, file); + + // Raise a new file created event + FileCreatedAsyncEvent fileCreatedAsyncEvent = new FileCreatedAsyncEvent(); + fileCreatedAsyncEvent.setDocument(document); + fileCreatedAsyncEvent.setFile(file); + AppContext.getInstance().getAsyncEventBus().post(fileCreatedAsyncEvent); // Always return ok JSONObject response = new JSONObject(); diff --git a/docs-web/src/prod/resources/config.properties b/docs-web/src/prod/resources/config.properties index 299540e8..680c36e5 100644 --- a/docs-web/src/prod/resources/config.properties +++ b/docs-web/src/prod/resources/config.properties @@ -1,3 +1,3 @@ api.current_version=${project.version} api.min_version=1.0 -db.version=4 \ No newline at end of file +db.version=5 \ No newline at end of file diff --git a/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java b/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java index c685a146..d718ef31 100644 --- a/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java +++ b/docs-web/src/test/java/com/sismics/docs/rest/TestDocumentResource.java @@ -50,6 +50,7 @@ public class TestDocumentResource extends BaseJerseyTest { postParams.add("title", "My super document 1"); postParams.add("description", "My super description for document 1"); postParams.add("tags", tag1Id); + postParams.add("language", "eng"); long create1Date = new Date().getTime(); postParams.add("create_date", create1Date); response = documentResource.put(ClientResponse.class, postParams); @@ -80,6 +81,7 @@ public class TestDocumentResource extends BaseJerseyTest { JSONArray tags = documents.getJSONObject(0).getJSONArray("tags"); Assert.assertTrue(documents.length() == 1); Assert.assertEquals(document1Id, documents.getJSONObject(0).getString("id")); + Assert.assertEquals("eng", documents.getJSONObject(0).getString("language")); Assert.assertEquals(1, tags.length()); Assert.assertEquals(tag1Id, tags.getJSONObject(0).getString("id")); Assert.assertEquals("SuperTag", tags.getJSONObject(0).getString("name")); @@ -135,18 +137,30 @@ public class TestDocumentResource extends BaseJerseyTest { Assert.assertEquals(document1Id, documents.getJSONObject(0).getString("id")); Assert.assertEquals(true, documents.getJSONObject(0).getBoolean("shared")); - // Search documents with multiple criteria + // Search documents by language documentResource = resource().path("/document/list"); documentResource.addFilter(new CookieAuthenticationFilter(document1Token)); getParams = new MultivaluedMapImpl(); - getParams.putSingle("search", "after:2010 before:2040-08 tag:super shared:yes for"); + getParams.putSingle("search", "lang:eng"); + response = documentResource.queryParams(getParams).get(ClientResponse.class); + json = response.getEntity(JSONObject.class); + Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); + documents = json.getJSONArray("documents"); + Assert.assertTrue(documents.length() == 1); + Assert.assertEquals(document1Id, documents.getJSONObject(0).getString("id")); + Assert.assertEquals("eng", documents.getJSONObject(0).getString("language")); + + // Search documents with multiple criteria + documentResource = resource().path("/document/list"); + documentResource.addFilter(new CookieAuthenticationFilter(document1Token)); + getParams = new MultivaluedMapImpl(); + getParams.putSingle("search", "after:2010 before:2040-08 tag:super shared:yes lang:eng for"); response = documentResource.queryParams(getParams).get(ClientResponse.class); json = response.getEntity(JSONObject.class); Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); documents = json.getJSONArray("documents"); Assert.assertTrue(documents.length() == 1); Assert.assertEquals(document1Id, documents.getJSONObject(0).getString("id")); - Assert.assertEquals(true, documents.getJSONObject(0).getBoolean("shared")); // Search documents (nothing) documentResource = resource().path("/document/list"); @@ -181,6 +195,17 @@ public class TestDocumentResource extends BaseJerseyTest { documents = json.getJSONArray("documents"); Assert.assertTrue(documents.length() == 0); + // Search documents (nothing) + documentResource = resource().path("/document/list"); + documentResource.addFilter(new CookieAuthenticationFilter(document1Token)); + getParams = new MultivaluedMapImpl(); + getParams.putSingle("search", "lang:fra"); + response = documentResource.queryParams(getParams).get(ClientResponse.class); + json = response.getEntity(JSONObject.class); + Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); + documents = json.getJSONArray("documents"); + Assert.assertTrue(documents.length() == 0); + // Get a document documentResource = resource().path("/document/" + document1Id); documentResource.addFilter(new CookieAuthenticationFilter(document1Token)); diff --git a/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java b/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java index 1a71e53a..1a70f0b2 100644 --- a/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java +++ b/docs-web/src/test/java/com/sismics/docs/rest/TestFileResource.java @@ -42,6 +42,7 @@ public class TestFileResource extends BaseJerseyTest { documentResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken)); MultivaluedMapImpl postParams = new MultivaluedMapImpl(); postParams.add("title", "File test document 1"); + postParams.add("language", "eng"); ClientResponse response = documentResource.put(ClientResponse.class, postParams); Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); JSONObject json = response.getEntity(JSONObject.class); @@ -52,7 +53,7 @@ public class TestFileResource extends BaseJerseyTest { WebResource fileResource = resource().path("/file"); fileResource.addFilter(new CookieAuthenticationFilter(file1AuthenticationToken)); FormDataMultiPart form = new FormDataMultiPart(); - InputStream file = this.getClass().getResourceAsStream("/file/PIA00452.jpg"); + InputStream file = this.getClass().getResourceAsStream("/file/Einstein-Roosevelt-letter.png"); FormDataBodyPart fdp = new FormDataBodyPart("file", new BufferedInputStream(file), MediaType.APPLICATION_OCTET_STREAM_TYPE); @@ -87,7 +88,7 @@ public class TestFileResource extends BaseJerseyTest { Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); InputStream is = response.getEntityInputStream(); byte[] fileBytes = ByteStreams.toByteArray(is); - Assert.assertEquals(163510, fileBytes.length); + Assert.assertEquals(292641, fileBytes.length); // Get the thumbnail data fileResource = resource().path("/file/" + file1Id + "/data"); @@ -98,7 +99,7 @@ public class TestFileResource extends BaseJerseyTest { Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); is = response.getEntityInputStream(); fileBytes = ByteStreams.toByteArray(is); - Assert.assertEquals(41935, fileBytes.length); + Assert.assertEquals(34050, fileBytes.length); // Get all files from a document fileResource = resource().path("/file/list"); diff --git a/docs-web/src/test/java/com/sismics/docs/rest/TestShareResource.java b/docs-web/src/test/java/com/sismics/docs/rest/TestShareResource.java index 0086a49d..443908c3 100644 --- a/docs-web/src/test/java/com/sismics/docs/rest/TestShareResource.java +++ b/docs-web/src/test/java/com/sismics/docs/rest/TestShareResource.java @@ -42,6 +42,7 @@ public class TestShareResource extends BaseJerseyTest { documentResource.addFilter(new CookieAuthenticationFilter(share1AuthenticationToken)); MultivaluedMapImpl postParams = new MultivaluedMapImpl(); postParams.add("title", "File test document 1"); + postParams.add("language", "eng"); ClientResponse response = documentResource.put(ClientResponse.class, postParams); Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); JSONObject json = response.getEntity(JSONObject.class); diff --git a/docs-web/src/test/java/com/sismics/docs/rest/TestTagResource.java b/docs-web/src/test/java/com/sismics/docs/rest/TestTagResource.java index f1bf30fb..cd8d1ecf 100644 --- a/docs-web/src/test/java/com/sismics/docs/rest/TestTagResource.java +++ b/docs-web/src/test/java/com/sismics/docs/rest/TestTagResource.java @@ -66,6 +66,7 @@ public class TestTagResource extends BaseJerseyTest { postParams = new MultivaluedMapImpl(); postParams.add("title", "My super document 1"); postParams.add("tags", tag3Id); + postParams.add("language", "eng"); response = documentResource.put(ClientResponse.class, postParams); Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); json = response.getEntity(JSONObject.class); @@ -76,6 +77,7 @@ public class TestTagResource extends BaseJerseyTest { postParams = new MultivaluedMapImpl(); postParams.add("title", "My super document 1"); postParams.add("tags", tag4Id); + postParams.add("language", "eng"); response = documentResource.put(ClientResponse.class, postParams); Assert.assertEquals(Status.OK, Status.fromStatusCode(response.getStatus())); json = response.getEntity(JSONObject.class); diff --git a/docs-web/src/test/resources/file/Einstein-Roosevelt-letter.png b/docs-web/src/test/resources/file/Einstein-Roosevelt-letter.png new file mode 100644 index 00000000..f7d1f242 Binary files /dev/null and b/docs-web/src/test/resources/file/Einstein-Roosevelt-letter.png differ