4x speedup of recursive rmdir in std.file

Jay Norwood jayn at prismnet.com
Sat Feb 4 15:36:15 PST 2012


It would be good if the std.file operations used the D multi-
thread features, since you've done such a nice job of making them
easy.   I hacked up your std.file recursive remove and got a 4x
speed-up on a win7 system with corei7 using the examples from the
D programming language book.  Code is below with a hard-coded file
I was using for test.  I'm just learning this, so I know you can
do better ...

Delete time dropped from 1minute 5 secs to less than 15 secs.
This was on an ssd drive.

module main;

import std.stdio;
import std.file;
import std.datetime;
import std.concurrency;
const int THREADS = 16;
int main(string[] argv)
{
   writeln("removing H:/pa10_120130/xx8");
	auto st1 = Clock.currTime(); //Current time in local time.
	rmdirRecurse2("H:/pa10_120130/xx8");
 	auto st2 = Clock.currTime(); //Current time in local time.
	auto dif = st2  - st1 ;
	auto ts= dif.toString();
	writeln("time:");
	writeln(ts);
	writeln("finished !");
   return 0;
}
void rmdirRecurse2(in char[] pathname){
    DirEntry de = dirEntry(pathname);
    rmdirRecurse2(de);
}
void rmdirRecurse2(ref DirEntry de){
	if(!de.isDir)
		throw new FileException( de.name, " is not a
directory");
	if(de.isSymlink())
		remove(de.name);
	else    {
		Tid tid[THREADS];
		int i=0;
        for(;i<THREADS;i++){
			tid[i]= spawn(&fileRemover);
		}
		Tid tidd = spawn(&dirRemover);

		// all children, recursively depth-first
	    i=0;
		foreach(DirEntry e; dirEntries(de.name,
SpanMode.depth, false))        {
			string nm = e.name;
            attrIsDir(e.linkAttributes) ? tidd.send(nm)  : tid
[i].send(nm),i=(i+1)%THREADS;
		}

        // wait for the THREADS threads to complete their file
removes and acknowledge
		// receipt of the tid
		for (i=0;i<THREADS;i++){
			tid[i].send(thisTid);
			receiveOnly!Tid();
		}
		tidd.send(thisTid);
		receiveOnly!Tid();

		// the dir itself
		rmdir(de.name);
	}
}
	void fileRemover() {
		for(bool running=true;running;){
		receive(
				(string s) {
					remove(s);
				}, // remove the files
				(Tid x) {
					x.send(thisTid);
					running=false;

				} // this is the terminator
				);
		}
	}

	void dirRemover() {
		string[] dirs;
		for(bool running=true;running;){
			receive(
					(string s) {
						dirs~=s;
					},
					(Tid x) {
						foreach(string
d;dirs){
							rmdir(d);
						}
						x.send(thisTid);
						running = false;
					}
					);
		}
	}




More information about the Digitalmars-d-announce mailing list