Nested Stencil - OpenGL ES - c

Question

I have a 2D node scene graph that I'm trying to 'nest' stencil clipping in.
I was thinking what I could do is when drawing the stencil, increment any pixel it writes to by 1, and keep track of what the current 'layer' is that I'm on.
Then when drawing, only write pixel data to the color buffer if the value of the stencil at that pixel is >= the current layer #.
This is the code I have now. It doesn't quite work. Where am I messing up?
First I call SetupStencilForMask().
Then draw stencil primitives.
Next, call SetupStencilForDraw().
Now draw actual imagery
When done with a layer, call DisableStencil().
Edit: Updated with solution. It doesn't work for individual items on the same layer, but otherwise is fine.
Found a great article on how to actually pull this off, although it's fairly limited.
http://cranialburnout.blogspot.com/2014/03/nesting-and-overlapping-translucent.html
// glClear(GL_STENICL_BIT) at start of each draw frame
static int stencilLayer = 0;
void SetupStencilForMask(void)
{
if (stencilLayer == 0)
glEnable(GL_STENCIL_TEST);
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
glStencilFunc(GL_LESS, stencilLayer, 0xff);
glStencilOp(GL_INCR, GL_KEEP, GL_KEEP);
glStencilMask(0xff);
if (stencilLayer == 0)
glClear(GL_STENCIL_BUFFER_BIT);
stencilLayer++;
}
void SetupStencilForDraw()
{
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
glStencilFunc(GL_EQUAL, stencilLayer, 0xff);
glStencilMask(0x00);
}
void DisableStencil(void)
{
if (--stencilLayer == 0)
glDisable(GL_STENCIL_TEST);
}

Answer 1

I have figured out the way to do this in libgdx. I'm not sure you still need this, but for future reference here is the code:
/**
* Start cropping
*
* #param cropMask
* Mask plane
*/
public void startCropping(Plane cropMask) {
// Check if there is active masking group
if (activeCropMaskGroup == null) {
// Create new one
activeCropMaskGroup = new CropMaskingGroupDescriptor(cropMask);
} else {
// Increase hierarchy level
activeCropMaskGroup.increaseHierachy(cropMask);
}
}
/** End cropping */
public void endCropping() throws IllegalStateException {
// Check if there is active group mask
if (activeCropMaskGroup == null) {
throw new IllegalStateException("Call start cropping before this!");
}
if (activeCropMaskGroup.getHierachy() > 0) {
activeCropMaskGroup.decreaseHierachy();
} else {
// Finish setup of crop data
cropMaskGroups.add(activeCropMaskGroup);
activeCropMaskGroup = null;
}
}
/** Crop registered planes for cropping */
private void cropRender(CropMaskingGroupDescriptor cropMaskGroupDescriptor) {
// Draw mask to stencil buffer
Gdx.gl.glClear(GL20.GL_STENCIL_BUFFER_BIT);
// setup drawing to stencil buffer
Gdx.gl20.glEnable(GL20.GL_STENCIL_TEST);
// Number of registered hierarchy levels
int hierarchyLevels = cropMaskGroupDescriptor.getRegisteredBatch().size();
// Loop trough hierarchy
for (int hierarchyLevel = 0; hierarchyLevel < hierarchyLevels; hierarchyLevel++) {
Gdx.gl20.glStencilFunc(GL20.GL_ALWAYS, 0x1, 0xffffffff);
Gdx.gl20.glStencilOp(GL20.GL_INCR, GL20.GL_INCR, GL20.GL_INCR);
Gdx.gl20.glColorMask(false, false, false, false);
Gdx.gl20.glDepthMask(false);
// Draw mask with decal batch
cropMaskBatch.add(((NativePlane) cropMaskGroupDescriptor.getCroppingMasks().get(hierarchyLevel)).getPlane());
cropMaskBatch.flush();
// fix stencil buffer, enable color buffer
Gdx.gl20.glColorMask(true, true, true, true);
Gdx.gl20.glDepthMask(true);
Gdx.gl20.glStencilOp(GL20.GL_KEEP, GL20.GL_KEEP, GL20.GL_KEEP);
// draw where pattern has been drawn
Gdx.gl20.glStencilFunc(GL20.GL_LEQUAL, hierarchyLevel + 1, 0xffffffff);
// Loop trough registered masked layers and found which one belongs
// to
// current hierarchy level
for (int i = 0; i < cropMaskGroupDescriptor.getMaskedLayers().size(); i++) {
if (cropMaskGroupDescriptor.getMaskedLayers().get(i).getHierarchyId() == hierarchyLevel) {
Plane plane = cropMaskGroupDescriptor.getMaskedLayers().get(i).getMaskedPlane();
cropMaskGroupDescriptor.getRegisteredBatch().get(hierarchyLevel).add(((NativePlane) plane).getPlane());
}
}
cropMaskGroupDescriptor.getRegisteredBatch().get(hierarchyLevel).flush();
}
Gdx.gl20.glDisable(GL20.GL_STENCIL_TEST);
}
And inner class inside renderer module.
/**
* Cropped layer descriptor
*
* #author Veljko Ilkic
*
*/
private class CropMaskLayerDescriptor {
/** Layer that needs to be masked */
private Plane maskedPlane;
/** Hierarchy level in which belongs */
private int hierarchyId = 0;
/** Constructor 1 */
public CropMaskLayerDescriptor(Plane maskedPlane, int hierarchyId) {
this.maskedPlane = maskedPlane;
this.hierarchyId = hierarchyId;
}
public Plane getMaskedPlane() {
return maskedPlane;
}
public int getHierarchyId() {
return hierarchyId;
}
}
/**
* Crop masking group descriptor class
*
* #author Veljko Ilkic
*
*/
private class CropMaskingGroupDescriptor {
/** Crop mask */
private ArrayList<Plane> croppingMasks = new ArrayList<Plane>();
/** Planes that will be masked by crop mask */
private ArrayList<CropMaskLayerDescriptor> maskedLayers = new ArrayList<Renderer.CropMaskLayerDescriptor>();
/** Batch for drawing masked planes */
private ArrayList<DecalBatch> hierarchyBatches = new ArrayList<DecalBatch>();
private int activeHierarchyLayerId = 0;
/** Constructor 1 */
public CropMaskingGroupDescriptor(Plane topLevelCropMask) {
// Create batch for top level hierarchy
hierarchyBatches.add(new DecalBatch(new CameraGroupStrategy(perspectiveCamera)));
// Register top level crop mask
croppingMasks.add(topLevelCropMask);
}
/** Increase hierarchy level of the group */
public void increaseHierachy(Plane hierarchyCropMask) {
activeHierarchyLayerId++;
// Create individual batch for hierarchy level
hierarchyBatches.add(new DecalBatch(new CameraGroupStrategy(perspectiveCamera)));
// Register crop mask for current hierarchy level
croppingMasks.add(hierarchyCropMask);
}
/** Decrease hierarchy group */
public void decreaseHierachy() {
activeHierarchyLayerId--;
}
/** Get current hierarchy level */
public int getHierachy() {
return activeHierarchyLayerId;
}
/** Register plane for masking */
public void registerLayer(Plane maskedPlane) {
hierarchyBatches.get(activeHierarchyLayerId).add(((NativePlane) maskedPlane).getPlane());
maskedLayers.add(new CropMaskLayerDescriptor(maskedPlane, activeHierarchyLayerId));
}
/** Get all registered batched */
public ArrayList<DecalBatch> getRegisteredBatch() {
return hierarchyBatches;
}
/** Get registered cropping masks */
public ArrayList<Plane> getCroppingMasks() {
return croppingMasks;
}
/** Get layer that should be masked */
public ArrayList<CropMaskLayerDescriptor> getMaskedLayers() {
return maskedLayers;
}
/** Dispose */
public void dispose() {
for (int i = 0; i < hierarchyBatches.size(); i++) {
hierarchyBatches.get(i).dispose();
hierarchyBatches.set(i, null);
}
hierarchyBatches.clear();
}
}
Hope this helps.