How do I cast to from byte[] <-> double for making a small assembler language VM to work?

[Enjoys Math via Digitalmars-d-learn]

class OpCode
{
private:
	byte[] bytes_;

public:
	void opCall(Program program) const;

	byte[] bytes() const {
		return bytes_.dup;
	}
}



class AddD : OpCode
{
private:
	uint d, s;

public:
	this(uint dst, uint src) {
		d = dst;
		s = src;
	}

	override void opCall(Program p) const
	{
		p[d..d+8] = cast(byte[])(cast(double)p[d..d+8] + 
cast(double)p[s..s+8]);
	}
}


---

The cast at the bottom gives a compiler error (can't cast byte[] 
to double).