MNEMOSYNE is a three years research project co-funded by the MICC – University of Florence and the Tuscany – European Social Fund. The project is about the study and experimentation of smart environments for the protection and promotion of artistic and cultural heritage. It adopts natural interaction paradigms for access and manipulation of multimedia information and rely on the passive analysis of visitors behaviors [1] to estimate their interests.
The current vision techniques applied in cultural heritage environments, such as museums, usually provide solutions for protection by detecting situations of potential risk and then notify operators responsible for safety. The idea of the project is to use techniques derived from video-surveillance scenarios to design an automatic profiling system capable of understanding the personal interest of many visitors.
The computer vision system monitors and analyzes the movements and behaviors of visitors in the museum (through the use of fixed cameras) in order to extract a profile of interests for each visitors. This profile of interest is then used to personalize the delivery of in-depth multimedia content enabling an augmented museum experience. Visitors interact with the multimedia content through a large interactive table installed inside the museum. The project also includes the integration of mobile devices (such as smartphones or tablets) offering a take-away summary of the visitor experience and suggesting possible theme-related paths in the collection of the museum or in other places of the city.
My work in this project was to build the back-end and computer vision systems, especially the passive profiling of the visitors inside the museum, with related research themes such as Re-Identification [2, 3, 4], Tracking [5], and Person Detection [6]. The whole MNEMOSYNE system [7] is currently under deployment at the Bargello Museum in Florence.
Related publications
![[pdf]](https://www.sveborkaraman.com/wp-content/plugins/papercite/img/pdf.png)
![[doi]](https://www.sveborkaraman.com/wp-content/plugins/papercite/img/external.png){kind=small}
S. Karaman, A. D. Bagdanov, G. D’Amico, L. Landucci, A. Ferracani, D. Pezzatini, and A. Del Bimbo, “Passive Profiling and Natural Interaction Metaphors for Personalized Multimedia Museum Experiences,” in MM4CH’13 – New Trends in Image Analysis and Processing – ICIAP 2013, Naples, Italy: Springer, 2013, p. 247–256. [Bibtex]
@incollection{karaman2013passive,
title = {Passive Profiling and Natural Interaction Metaphors for Personalized Multimedia Museum Experiences},
author = {Karaman, Svebor and Bagdanov, Andrew D and D’Amico, Gianpaolo and Landucci, Lea and Ferracani, Andrea and Pezzatini, Daniele and Del Bimbo, Alberto},
booktitle = {MM4CH'13 - New Trends in Image Analysis and Processing -- ICIAP 2013},
doi = {10.1007/978-3-642-41190-8_27},
pages = {247--256},
address = {Naples, Italy},
year = {2013},
note={Oral Presentation},
publisher = {Springer}
}[Bibtex]
@incollection{karamanIdInf2012,
isbn={978-3-642-33862-5},
booktitle={Computer Vision – ECCV 2012. Workshops and Demonstrations},
volume={7583},
series={Lecture Notes in Computer Science},
editor={Fusiello, Andrea and Murino, Vittorio and Cucchiara, Rita},
doi={10.1007/978-3-642-33863-2_44},
title={Identity Inference: Generalizing Person Re-identification Scenarios},
url={http://dx.doi.org/10.1007/978-3-642-33863-2_44},
publisher={Springer Berlin Heidelberg},
author={Karaman, Svebor and Bagdanov, Andrew D.},
pages={443-452},
address = {Firenze, Italy},
note={Oral Presentation. Best Paper Award},
year={2012}
}[Bibtex]
@incollection{KaramanReID2014,
author = {Karaman, Svebor and Lisanti, Giuseppe and Bagdanov, Andrew D. and Del Bimbo, Alberto},
title = {From Re-identification to Identity Inference: Labeling Consistency by Local Similarity Constraints},
booktitle = {Person Re-Identification},
series = {Advances in Computer Vision and Pattern Recognition},
editor = {Gong, Shaogang and Cristani, Marco and Yan, Shuicheng and Loy, Chen Change},
isbn = {978-1-4471-6295-7},
doi = {10.1007/978-1-4471-6296-4_14},
url = {http://dx.doi.org/10.1007/978-1-4471-6296-4_14},
publisher = {Springer London},
keywords = {Re-identification; Identity inference; Conditional random fields; Video surveillance},
pages = {287-307},
language = {English},
year = {2014}
}[Bibtex]
@article{karaman2014leveraging,
title = "Leveraging local neighborhood topology for large scale person re-identification ",
journal = "Pattern Recognition ",
volume = "47",
number = "12",
pages = "3767 - 3778",
year = "2014",
note = "",
issn = "0031-3203",
doi = "10.1016/j.patcog.2014.06.003",
url = "http://www.sciencedirect.com/science/article/pii/S0031320314002258",
author = "Svebor Karaman and Giuseppe Lisanti and Andrew D. Bagdanov and Alberto Del Bimbo",
keywords = "Re-Identification",
keywords = "Conditional Random Field",
keywords = "Semi-supervised",
keywords = "\{ETHZ\}",
keywords = "\{CAVIAR\}",
keywords = "3DPeS",
keywords = "\{CMV100\} "
}[Bibtex]
@inproceedings{DBLMKD13,
author = {Bagdanov, Andrew D. and Del Bimbo, Alberto and Di Fina, Dario and Karaman, Svebor and Lisanti, Giuseppe and Masi, Iacopo},
title = {Multi-Target Data Association using Sparse Reconstruction},
booktitle = {Proc. of International Conference on Image Analysis and Processing (ICIAP)},
year = {2013},
address = {Naples, Italy},
pages = {239-248},
note={Poster},
doi = {10.1007/978-3-642-41184-7_25},
publisher = {Springer Berlin Heidelberg},
keywords = {Data association; multi-target tracking; sparse methods; video surveillance},
url = {http://www.micc.unifi.it/publications/2013/DBLMKD13}
}[Bibtex]
@InProceedings{bartoliicpr2014,
author = {Bartoli, Federico and Lisanti, Giuseppe and Karaman, Svebor and Bagdanov, Andrew D. and Del Bimbo, Alberto},
title = {Unsupervised scene adaptation for faster multi-scale pedestrian detection},
note = {Oral presentation},
booktitle = {22nd International Conference on Pattern Recognition (ICPR)},
address = {Stockholm, Sweden},
year = {2014}
}[Bibtex]
@article{karaman2014mtap,
year={2014},
issn={1380-7501},
journal={Multimedia Tools and Applications},
doi={10.1007/s11042-014-2192-y},
title={Personalized multimedia content delivery on an interactive table by passive observation of museum visitors},
url={http://dx.doi.org/10.1007/s11042-014-2192-y},
publisher={Springer US},
keywords={Computer vision; Video surveillance; Cultural heritage; Multimedia museum; Personalization; Natural interaction; Passive profiling},
author={Karaman, Svebor and Bagdanov, AndrewD. and Landucci, Lea and D’Amico, Gianpaolo and Ferracani, Andrea and Pezzatini, Daniele and Del Bimbo, Alberto},
pages={1-25},
language={English}
}