Not all Images loaded while carving through a File System ?

Hello,

I'm sorry for posting an extra Topic, about a former Code. But the Title misleaded.

To the Problem. The Array contains around 60000 Paths. But the resulting Images are only about 1500.

I tried different for each function loops, parallel or ordered. Using some System.out.prints to show the count.
It's still the same it shows around 2000 pictures on the Window.
The count shows around 60000 Image Files and 2000 loaded Images.

The Log Files :
(All Pictures (around 8000 kB)),

(Showed Pictures (around 170 kB)),

((No Access (around 17 kB size)).


How could I figure out where the problems arises, or if it's just in the Nature of the System (Java or Windows) ?

class Task2 extends SwingWorker<Void, Void> {        @Override        public Void doInBackground() throws InterruptedException, IOException, URISyntaxException {            BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>();            ExecutorService executor = Executors.newFixedThreadPool(100);// Your task logic here                    IntStream stream = IntStream.range(0, filteredPaths.size());                    System.out.println("Filtered Files Array Size : " + "  " + filteredPaths.size());                    stream.forEach(zp -> {                        System.out.println("Images Count : " + "  " + zp);                        try {                            inStream = new FileInputStream(String.valueOf(filteredPaths.get(zp).toFile()));                        } catch (FileNotFoundException e) {                            throw new RuntimeException(e);                        }                        imageIcons = zp;                        try {                            img = ImageIO.read(inStream);                        } catch (IOException iioe) {}                        if (img == null) {                            imageArr.add(new ImageIcon(""));                            thumbLabels.add(new JLabel((imageArr.get(zp))));                            thumbLabels.get(zp).setToolTipText(filteredPaths.get(zp).toString());                            try {                                System.out.println("Invalid Images !");                                // invalid Pictures                                w3.write(filteredPaths.get(zp).toString() + "\r\n");                            } catch (IOException e) {                                throw new RuntimeException(e);                            }                        } else {                            executor.submit(() -> {                                imageArr.add(new ImageIcon(img.getScaledInstance(51, 51, Image.SCALE_SMOOTH)));                                thumbLabels.add(new JLabel(imageArr.get(zp)));                                thumbLabels.get(zp).setToolTipText(filteredPaths.get(zp).toString());                            });                            try {                                // readable Pictures                                w2.write(filteredPaths.get(zp).toString() + "\r\n");                            } catch (IOException e) {                                throw new RuntimeException(e);                            }                        } try {                            inStream.close();                        } catch (IOException e) {                            throw new RuntimeException(e);                        }                    });            try {                w2.close();                w3.close();            } catch (IOException e) {                throw new RuntimeException(e);            }            executor.shutdown();                    return null;        }        protected void done() {            System.out.println("Thumb Labels Size :" + "  " + thumbLabels.size());            task3 = new Task3();            task3.addPropertyChangeListener(this);            task3.execute();        }        public void addPropertyChangeListener(Runnable runnable) {        }    }    class Task3 extends SwingWorker<Void, Void> {        @Override        public Void doInBackground() throws InterruptedException, IOException, URISyntaxException {            IntStream stream = IntStream.range(0, filteredPaths.size());            stream.forEach(zh -> {                grid.add(thumbLabels.get(zh));                gridIcons = zh;                p.revalidate();                p.repaint();                grid.revalidate();                grid.repaint();                scrollP.revalidate();                scrollP.repaint();            });            return null;        }        protected void done() {            tm1 = new SimpleDateFormat("HH:mm:ss").format(new Date().getTime());            SimpleDateFormat format = new SimpleDateFormat("HH:mm:ss");            System.out.println("Check start time : " + tm);            System.out.println("Check end time : " + tm1);            Date date1 = null;            try {                date1 = format.parse(tm);            } catch (ParseException e) {                throw new RuntimeException(e);            }            Date date2 = null;            try {                date2 = format.parse(tm1);            } catch (ParseException e) {                throw new RuntimeException(e);            }            long difference = date2.getTime() - date1.getTime();            long diffMinutes = difference / (60 * 1000) % 60;            System.out.println("All Images loaded !");            System.out.println("All ImageIcons :" + "  " + imageIcons);            System.out.println("All GridIcons :" + "  " + gridIcons);            System.out.println("Time elapsed : " + "  " + diffMinutes + "  " + "minutes");        }        public void addPropertyChangeListener (Runnable runnable){        }    } }

Thanks

Best reagrds,
John

I would definitely fix line 18 so that it doesn't ignore exceptions. You fixed all the other try-catches to throw a RuntimeException, it looks like you just missed fixing this one.

You don't need so many different try-catch blocks - especially when they all (or, erroneously, all but one) do the same thing.  Most all of those could be handled by one big block that covers the main body of the doInBackground() method.  

I would add something like an e.printStackTrace() (or, better, a proper log message) in the catch block.  Normally, you shouldn't do this if you are also rethrowing an exception, as it can lead to excess logged messages.  But that assumes that there's some catch block higher up that will eventually log the error.  If you're not seeing error messages here, it can be good to make extra sure you're not losing them.  Especially when using multi-threading in some way.  You may find that the work is being done in a worker thread that doesn't log errors as you'd expect.

On a related note, the executor.submit() method returns a Future object, which you are currently ignoring.  I recommend collecting these and putting them in a List for later review.  You may find that these Futures are erroring out, or still running, or something else.

Your current code seems to still be doing a lot of the work in one thread, in doInBackground().  Only the code in the executor.submit() lambda expression is actually being delegated to other threads.  That may be a lesser problem, if you're trying to understand why not enough images are being created.  Then again, is it possible that more are still being created, but it's still happening slowly, in the doInBackground() method?  This goes back to the problem that since you're ignoring the returned Future objects, you don't have any way of knowing if this has actually completed yet.  Maybe your GUI just has no way of knowing that more images have been created, later on?

In your previous posting you asked if only certain file types can be displayed.

ImageIO uses a "reader" to read files of a given file type. Not all file types are supported.

You can check which file types are supported by using:

System.out.println(Arrays.asList(ImageIO.getReaderFormatNames()));

Your previous question also mentioned memory issues. It is not very efficient to use a JLabel to display 60,000 images.

Swing was designed to be used with renderers. So, for example you can use a JList to display the images. In this case only a single JLabel is used to render each image.

I have a very simple  "Thumbnail App" that displays images in a JList. It only loads images from a single directory, so you would need to modify the code to use your filtered list of files. It consists of 5 classes I could post if you are interested.

Hello,

@Paul Clapham, yes thank you for the Advise, I will reduce the Exceptions and move it to one.
@Mike Simmons, yes thank you for the Advise, I will try to use ExecutorService along with FutureTask.
@Rob Camick, yes thank too for the precious Advise, I will change it accordingly. And I don't mind to send me your Project, I'm glad to test and practice with it.

Sorry, nontheless I read few Books about Java Programming and some still to read I missed to read more about Concurrency and deepen other Stuff, to understand the whole thing.

After tinkering a while, I'm not 100% sure if the Code respects the Rules for this Case. Here my actual Code. It runs, but all Images are recognized as Null ?

private static ExecutorService executor = Executors.newFixedThreadPool(100);    private static Callable<String> callableTask = new MyCallable();    private static FutureTask<String> futureTask = new FutureTask<>(callableTask);   static class MyCallable implements Callable<String> {            @Override            public String call() throws Exception { // To do ..... }


executor.submit(futureTask);                try {                    String result = futureTask.get();                    System.out.println(result);                } catch (Exception e) {                    e.printStackTrace();                } finally {                    executor.shutdown();                }


Vector<JPanel> items = new Vector<>(); for (int i = 0; i < filteredPaths.size(); i++) {                        try {                            if(img != null) {                                img = ImageIO.read(filteredPaths.get(i).toFile());                            panel = new JPanel(new FlowLayout(FlowLayout.LEFT));                            label = new JLabel(new ImageIcon(img));                                label.setToolTipText(filteredPaths.get(i).toString());                            panel.add(label);                            items.add(panel);                            panel.revalidate();                            panel.repaint();                                w2.write("Image okay : " + "  " + filteredPaths.get(i).toString() + "\r\n");                        } else {w1.write("Image Null : " + "  " + filteredPaths.get(i).toString() + "\r\n");}                            System.out.println("Count" + "  " + i);                        } catch (IOException e) {                            System.err.println("Error loading image: " + e.getMessage());                        }                    }                    JList<JPanel> list = new JList<>(items);                    list.setCellRenderer(new ImageListCellRenderer());                    frame.add(new JScrollPane(list));                    //frame.setVisible(true);                    frame.revalidate();                    frame.repaint();

static class ImageListCellRenderer implements ListCellRenderer<JPanel> {            public Component getListCellRendererComponent(JList<? extends JPanel> list, JPanel value, int index, boolean isSelected, boolean cellHasFocus) {                value.setBackground(isSelected ? Color.LIGHT_GRAY : Color.WHITE);                System.out.println("Cell Value : " + "  " + value);                return value;            }        }

Thanks.

Best Regards,
John

Here are the 5 classes.

At the bottom of the ThumbnailApp you should just need to change the filename that contains images you want to load. The app will load all the ".jpg" images found in the directory. This is the class you invoke to start the app.

The ThumbnailWorker using ImageIO to do sampling of the images to make reading of the images faster.

The JList is updated with the thumbnail image as it is loaded.

As you will see the code is similar to yours except is uses a JList to render the images, instead of using individual JLabels.

Have fun.

import java.awt.*; import javax.swing.*; public class EmptyIcon implements Icon {    private int width;    private int height;    public EmptyIcon(int width, int height)    {        this.width = width;        this.height = height;    }    public int getIconWidth()    {        return width;    }    public int getIconHeight()    {        return height;    }    public void paintIcon(Component c, Graphics g, int x, int y)    {    } }

import java.io.File; import javax.swing.Icon; public class Thumbnail { private File file; private Icon icon; public Thumbnail(File file, Icon icon) { this.file = file; this.icon = icon; } public Icon getIcon() {     return icon; } public void setIcon(Icon icon) { this.icon = icon; } public String getFileName() { return file.getName(); } }

import java.io.*; import java.util.concurrent.*; import java.awt.*; //import java.awt.datatransfer.*; import java.awt.event.*; import javax.swing.*; public class ThumbnailApp { private DefaultListModel<Thumbnail> model = new DefaultListModel<Thumbnail>(); private JList<Thumbnail> list = new JList<Thumbnail>(model); public ThumbnailApp() { } public JPanel createContentPane() { JPanel cp = new JPanel( new BorderLayout() ); list.setCellRenderer( new ThumbnailRenderer<Thumbnail>() ); list.setLayoutOrientation(JList.HORIZONTAL_WRAP); list.setVisibleRowCount(-1); Icon empty = new EmptyIcon(160, 160); // Thumbnail prototype = new Thumbnail(new File("PortugalSpain-000.JPG"), empty); Thumbnail prototype = new Thumbnail(new File(""), empty); list.setPrototypeCellValue( prototype ); // list.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); // list.setDragEnabled(true); // list.setTransferHandler(new ListIndexTransferHandler()); // list.setDropMode(DropMode.INSERT); cp.add(new JScrollPane( list ), BorderLayout.CENTER); return cp; } public void loadImages(File directory) { new Thread( () -> createThumbnails(directory) ).start(); } private void createThumbnails(File directory) { try { File[] files = directory.listFiles((d, f) -> {return f.endsWith(".JPG");}); int processors = Runtime.getRuntime().availableProcessors(); ExecutorService service = Executors.newFixedThreadPool( processors - 2 ); long start = System.currentTimeMillis(); for (File file: files) { Thumbnail thumbnail = new Thumbnail(file, null); model.addElement( thumbnail ); //        new ThumbnailWorker(file, model, model.size() - 1).execute(); service.submit( new ThumbnailWorker(file, model, model.size() - 1) ); } long duration = System.currentTimeMillis() - start; System.out.println(duration); service.shutdown(); } catch(Exception e) { e.printStackTrace(); } } private static void createAndShowGUI() { ThumbnailApp app = new ThumbnailApp(); JFrame frame = new JFrame("ListDrop"); frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); frame.setContentPane( app.createContentPane() ); frame.setSize(1600, 900); frame.setVisible(true); // File directory = new File("C:/Users/netro/Pictures/TravelSun/2019_01_Cuba"); File directory = new File("C:/someDirectory"); app.loadImages( directory ); } public static void main(String[] args) { javax.swing.SwingUtilities.invokeLater(() -> createAndShowGUI()); } }

import java.awt.Component; import javax.swing.*; import javax.swing.border.EmptyBorder; public class ThumbnailRenderer<E> extends JLabel implements ListCellRenderer<E> { public ThumbnailRenderer() { setOpaque(true); setHorizontalAlignment(CENTER); setVerticalAlignment(CENTER); setHorizontalTextPosition( JLabel.CENTER ); setVerticalTextPosition( JLabel.BOTTOM ); setBorder( new EmptyBorder(4, 4, 4, 4) ); } /* *  Display the Thumbnail Icon and file name. */ public Component getListCellRendererComponent(JList<? extends E> list, E value, int index, boolean isSelected, boolean cellHasFocus) { if (isSelected) { setBackground(list.getSelectionBackground()); setForeground(list.getSelectionForeground()); } else { setBackground(list.getBackground()); setForeground(list.getForeground()); } //Set the icon and filename Thumbnail thumbnail = (Thumbnail)value; setIcon( thumbnail.getIcon() ); setText( thumbnail.getFileName() ); setToolTipText( getText() ); System.out.println(thumbnail.getFileName()); return this; } }

import java.awt.*; import java.awt.image.*; import java.io.*; import java.util.Iterator; //import java.util.concurrent.*; import javax.imageio.*; import javax.imageio.stream.*; import javax.swing.*; public class ThumbnailWorker extends SwingWorker<Image, Void> { private File file; private DefaultListModel<Thumbnail> model; private int index; public ThumbnailWorker(File file, DefaultListModel<Thumbnail> model, int index) { this.file = file; this.model = model; this.index = index; } @Override protected Image doInBackground() throws IOException { // Image image = ImageIO.read( file ); Iterator<ImageReader> readers = ImageIO.getImageReadersByFormatName("jpg"); ImageReader reader = readers.next(); ImageReadParam irp = reader.getDefaultReadParam(); // irp.setSourceSubsampling(10, 10, 0, 0); irp.setSourceSubsampling(5, 5, 0, 0); ImageInputStream stream = new FileImageInputStream( file ); reader.setInput(stream); Image image = reader.read(0, irp); int width = 160; int height = 90; if (image.getHeight(null) > image.getWidth(null)) { width = 90; height = 160; } BufferedImage scaled = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); Graphics2D g2d = scaled.createGraphics(); g2d.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR); g2d.drawImage(image, 0, 0, width, height, null); g2d.dispose(); image = null; return scaled;   }   @Override   protected void done()   {   try   {   ImageIcon icon = new ImageIcon( get() );   Thumbnail thumbnail = model.get( index );   thumbnail.setIcon( icon );   model.set(index, thumbnail);   System.out.println("finished: " + file);   }   catch (Exception e)   {   e.printStackTrace();   }   } }

Hey Rob Camick,

Thank you very much for sharing your Code.

I changed the File Carving Part, to iterate recursively through all Folders.

 it works well.

Points to improve :

- Image Rendering (not all Images have the same Resolution, and some are without Image)

- Adding Filepath to the Tooltip.

- Create Logs for Files , accessed, denied, or error reading.

Best Regards,
John

Hello Rob Camick,

I changed a bit your Code to add ToolTip with Path instead of only File Name.
Added a SwingWorker Task with FileWalkTree and FileVisitor, to iterate over the whole File System.
Added more File Extension for Images.
Added another Code to create Thumbnails.

Now all Images are viewed. Not as your former Code, which is nontehless great.
However the File System carving takes 8% of CPU Ressource. And after carving it starts to create Thumbnails, which takes about 70-80% of CPU.
Well it works so far, but the percentage of CPU may indicate a problem when using more Programs at the same time on Windows.

I hope to make more of an effort to finish reading the book  (JUnit Tests) and to practice more. So getting more insight in such Problems that could arise.

Thanks.

import javax.imageio.ImageIO; import javax.imageio.ImageReadParam; import javax.imageio.ImageReader; import javax.imageio.stream.FileImageInputStream; import javax.imageio.stream.ImageInputStream; import javax.swing.*; import java.awt.*; import java.awt.geom.AffineTransform; import java.awt.image.BufferedImage; import java.io.File; import java.io.IOException; import java.util.Iterator; public class ThumbnailWorker extends SwingWorker<Image, Void> {    private File file;    private DefaultListModel<Thumbnail> model;    private int index;    public ThumbnailWorker(File file, DefaultListModel<Thumbnail> model, int index)    {        this.file = file;        this.model = model;        this.index = index;    }    @Override    protected Image doInBackground() throws IOException    {        Iterator<ImageReader> readers = null;        String[] readFormats = ImageIO.getReaderFormatNames(); //new        //System.out.println("Supported Read Formats:");        for (String format : readFormats) { // new reading all Formats            //System.out.println("- " + format);            readers = ImageIO.getImageReadersByFormatName(format);        }        ImageReader reader = readers.next();        ImageReadParam irp = reader.getDefaultReadParam();        irp.setSourceSubsampling(5, 5, 0, 0);        if(!file.canExecute() && !file.canRead()) { //founded Pictures            new ThumbnailApp2().w3.write(file.toString() + "\r\n"); // new for Log        }              int width = 130;        int height = 60;        BufferedImage originalImage = ImageIO.read(new File(file.toString())); // new        Image image = originalImage.getScaledInstance(width, height, Image.SCALE_SMOOTH); // new        if (image.getHeight(null) > image.getWidth(null))        {            width = 90;            height = 160;        }        BufferedImage thumbnailImage = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB); // new        thumbnailImage.getGraphics().drawImage(image, 0, 0, null); /// new            image = null;        return thumbnailImage;    } @Override    protected void done()    {        try        {            ImageIcon icon = new ImageIcon( get() );            Thumbnail thumbnail = model.get( index );            thumbnail.setIcon( icon );            model.set(index, thumbnail);            System.out.println("finished: " + file);        }        catch (Exception e)        {            e.printStackTrace();        }    }

In ThumbnailRenderer.java
 setText(thumbnail.getFileName());        setToolTipText(new File(thumbnail.getFileName()).getAbsolutePath());

Depending on the purpose of the usage of such a Program.
E.g. I find it useful for searching Files, supected, or missed.
In addition to this, adding a copy Function for the ToolTip Output etc.
Seems more towards a Forensic Tool, however fun because Windows Explorer might take more time to search in C: e.g. for .jpg.

Have a good Time,
John